app/pull-subscriptions.py

   1 #!/bin/sh
   2 ''':'
   3 . /opt/yt/venv/bin/activate
   4 exec python $0 "$@"
   5 ':'''
   6
   7 import sys
   8 import time
   9 import sqlite3
  10 from common import *
  11
  12 with sqlite3.connect(cf['global']['database']) as conn:
  13     c = conn.cursor()
  14     c.execute("""
  15           SELECT DISTINCT subscriptions.channel_id
  16             FROM subscriptions LEFT JOIN crawler
  17               ON subscriptions.channel_id = crawler.channel_id
  18 --         WHERE crawled_at IS NULL OR crawled_at < datetime('now', '-1 day')
  19         ORDER BY crawler.crawled_at
  20     """)
  21     for (channel_id,) in c.fetchall():
  22         if '-v' in sys.argv: sys.stderr.write(f'fetching {channel_id}\n')
  23
  24         xmlfeed = fetch_xml("channel_id", channel_id)
  25         ok = update_channel(conn, xmlfeed)
  26         if not ok:
  27             if '-v' in sys.argv: sys.stderr.write(f'FAILED: {channel_id}\n')
  28             # writing to the database failed, so we store the feed in a file for later analysis.
  29             with open('/tmp/pull-subscriptions.err', 'a') as f:
  30                 f.write(f"<!-- {time.ctime()} ({int(time.time())}) -->\n{xmlfeed}\n")
  31             continue
  32
  33         # update crawled_at timestamp:
  34         c.execute("""
  35                 INSERT OR REPLACE INTO crawler (channel_id)
  36                                 VALUES (?)
  37         """, (channel_id,))
  38
  39         conn.commit()
  40
  41         time.sleep(60)