app/utils/pull-subscriptions.py

   1 #!/bin/sh
   2 ''':'
   3 . /opt/yt/venv/bin/activate
   4 exec python $0 "$@"
   5 ':'''
   6
   7 import sys
   8 import time
   9 import sqlite3
  10 sys.path.insert(0,"/opt/yt/app/")
  11 from common.common import *
  12
  13 with sqlite3.connect(cf['global']['database']) as conn:
  14     c = conn.cursor()
  15     c.execute("""
  16           SELECT DISTINCT subscriptions.channel_id
  17             FROM subscriptions LEFT JOIN crawler
  18               ON subscriptions.channel_id = crawler.channel_id
  19 --         WHERE crawled_at IS NULL OR crawled_at < datetime('now', '-1 day')
  20         ORDER BY crawler.crawled_at
  21     """)
  22     for (channel_id,) in c.fetchall():
  23         if '-v' in sys.argv: sys.stderr.write(f'fetching {channel_id}\n')
  24
  25         xmlfeed = fetch_xml("channel_id", channel_id)
  26         if not xmlfeed and '-v' in sys.argv:
  27             sys.stderr.write(f'FETCH FAILED: {channel_id}\n')
  28
  29         try:
  30             update_channel(conn, xmlfeed)
  31         except:
  32             if '-v' in sys.argv: sys.stderr.write(f'STORE FAILED: {channel_id}\n')
  33             # writing to the database failed, so we store the feed in a file for later analysis.
  34             with open('/tmp/pull-subscriptions.err', 'ab') as f:
  35                 f.write(f"<!-- {time.ctime()} ({int(time.time())}) -->\n".encode('ascii'))
  36                 f.write(xmlfeed + b"\n")
  37             continue
  38
  39         # update crawled_at timestamp:
  40         c.execute("""
  41                 INSERT OR REPLACE INTO crawler (channel_id)
  42                                 VALUES (?)
  43         """, (channel_id,))
  44
  45         conn.commit()
  46
  47         time.sleep(60)