]>
git.gir.st - subscriptionfeed.git/blob - app/pull-subscriptions.py
3 . /opt/yt/venv/bin/activate
12 with sqlite3
.connect(cf
['global']['database']) as conn
:
15 SELECT subscriptions.channel_id
16 FROM subscriptions LEFT JOIN crawler
17 ON subscriptions.channel_id = crawler.channel_id
18 -- WHERE crawled_at IS NULL OR crawled_at < datetime('now', '-1 day')
19 ORDER BY crawler.crawled_at
21 for (channel_id
,) in c
.fetchall():
22 if '-v' in sys
.argv
: sys
.stderr
.write(f
'fetching {channel_id}\n')
24 xmlfeed
= fetch_xml("channel_id", channel_id
)
25 ok
= update_channel(conn
, xmlfeed
)
27 if '-v' in sys
.argv
: sys
.stderr
.write(f
'FAILED: {channel_id}\n')
28 # writing to the database failed, so we store the feed in a file for later analysis.
29 with
open('/tmp/pull-subscriptions.err', 'a') as f
:
30 f
.write(f
"<!-- {time.ctime()} ({int(time.time())}) -->\n{xmlfeed}\n")
33 # update crawled_at timestamp:
35 INSERT OR REPLACE INTO crawler (channel_id)