app/utils/pull-subscriptions.py

   1 #!/bin/sh
   2 ''':'
   3 . /opt/yt/venv/bin/activate
   4 exec python $0 "$@"
   5 ':'''
   6
   7 import sys
   8 import time
   9 import sqlite3
  10 sys.path.insert(0,"/opt/yt/app/")
  11 from common.common import *
  12
  13 with sqlite3.connect(cf['global']['database']) as conn:
  14     c = conn.cursor()
  15     c.execute("""
  16           SELECT DISTINCT subscriptions.channel_id
  17             FROM subscriptions LEFT JOIN crawler
  18               ON subscriptions.channel_id = crawler.channel_id
  19 --         WHERE crawled_at IS NULL OR crawled_at < datetime('now', '-1 day')
  20         ORDER BY crawler.crawled_at
  21     """)
  22     for (channel_id,) in c.fetchall():
  23         if '-v' in sys.argv: sys.stderr.write(f'fetching {channel_id}\n')
  24
  25         xmlfeed = fetch_xml("channel_id", channel_id)
  26         ok = update_channel(conn, xmlfeed)
  27         if not ok:
  28             if '-v' in sys.argv: sys.stderr.write(f'FAILED: {channel_id}\n')
  29             # writing to the database failed, so we store the feed in a file for later analysis.
  30             with open('/tmp/pull-subscriptions.err', 'a') as f:
  31                 f.write(f"<!-- {time.ctime()} ({int(time.time())}) -->\n{xmlfeed}\n")
  32             continue
  33
  34         # update crawled_at timestamp:
  35         c.execute("""
  36                 INSERT OR REPLACE INTO crawler (channel_id)
  37                                 VALUES (?)
  38         """, (channel_id,))
  39
  40         conn.commit()
  41
  42         time.sleep(60)