]>
git.gir.st - subscriptionfeed.git/blob - app/common/utils.py
3 . "`dirname "$0"`/../../venv/bin/activate"
16 'channel': 'channel_id',
17 'playlist': 'playlist_id',
20 def pull_subscriptions(verbose
=1, force_all
=False, limit
=-1):
22 Crawls youtube channels' RSS feeds and stores found videos in the database.
23 verbose: 0: completely silent; 1: warn on errors; 2: log all accessed feeds
24 force_all: fetch all known channels. otherwise only those not crawled in 24h
25 limit: limit number of processed feeds
27 with sqlite3
.connect(cf
['global']['database']) as conn
:
30 SELECT DISTINCT s.channel_id, type
31 FROM subscriptions AS s LEFT JOIN crawler AS c
32 ON s.channel_id = c.channel_id
33 WHERE ? OR IFNULL(crawled_at,0) < datetime('now', '-1 day')
36 """, (force_all
,limit
))
37 results
= c
.fetchall()
39 if verbose
>= 2 and not len(results
):
40 sys
.stderr
.write(f
'no feeds to update.\n')
42 for i
,(feed_id
, feed_type
) in enumerate(results
):
44 pull_feed(feed_id
, feed_type
, conn
, verbose
)
46 def pull_feed(feed_id
, feed_type
, conn
, verbose
):
50 sys
.stderr
.write(f
'fetching {feed_id}\n')
52 xmlfeed
= fetch_xml(feed_param
[feed_type
], feed_id
)
55 sys
.stderr
.write(f
'FETCH FAILED: {feed_id}\n')
59 update_channel(conn
, xmlfeed
)
62 sys
.stderr
.write(f
'STORE FAILED: {feed_id}\n')
63 # writing failed, so we store the feed in a file for later analysis.
64 with
open('/tmp/pull-subscriptions.err', 'ab') as f
:
65 f
.write(f
"<!-- {time.ctime()} ({int(time.time())}) -->\n"
67 f
.write(xmlfeed
+ b
"\n")
70 # update crawled_at timestamp:
72 INSERT OR REPLACE INTO crawler (channel_id)
80 def update_subscriptions(verbose
=1, force_all
=False, limit
=-1):
82 Refreshes the websub (pubsubhubhub) subscription requests for youtube feeds.
83 verbose: 0: completely silent; 1: warn on errors; 2: log all accessed feeds
84 limit: limit number of processed feeds
86 with sqlite3
.connect(cf
['global']['database']) as conn
:
89 SELECT DISTINCT s.channel_id, type
90 FROM subscriptions AS s LEFT JOIN websub AS w
91 ON s.channel_id = w.channel_id
92 WHERE ? OR IFNULL(subscribed_until,0) < datetime('now','+12 hours')
93 ORDER BY subscribed_until
95 """, (force_all
,limit
))
96 results
= c
.fetchall()
98 if verbose
>= 2 and not len(results
):
99 sys
.stderr
.write(f
'no feeds to update.\n')
101 for i
,(feed_id
, feed_type
) in enumerate(results
):
103 update_feed(feed_id
, feed_type
, verbose
)
105 def update_feed(feed_id
, feed_type
, verbose
):
106 webhook
= cf
['websub']['public_uri']
107 lease
= cf
['websub']['lease']
108 hmackey
= cf
['websub']['hmac_key']
111 sys
.stderr
.write(f
'updating {feed_id}\n')
113 feed_type
= feed_param
[feed_type
]
114 version
, timestamp
= "v1", int(time
.time())
115 nonce
= secrets
.token_urlsafe(16)
116 sig
= websub_url_hmac(hmackey
, feed_id
, timestamp
, nonce
)
117 r
= requests
.post("https://pubsubhubbub.appspot.com/subscribe", {
118 "hub.callback": f
"{webhook}/websub/{version}/{timestamp}/" + \
119 f
"{nonce}/{feed_id}/{sig}",
120 "hub.topic": f
"https://www.youtube.com/xml/feeds/videos.xml" + \
121 f
"?{feed_type}={feed_id}",
122 "hub.verify": "async",
123 "hub.mode": "subscribe",
124 "hub.lease_seconds": lease
,
125 "hub.secret": hmackey
,
129 sys
.stderr
.write(f
'FAILED {feed_id}: {r.text}\n')
135 if __name__
== '__main__':
136 if len(sys
.argv
) < 2 or sys
.argv
[1] not in ['pull','websub']:
137 sys
.stderr
.write(f
'Usage: YT_CONFIG=... {sys.argv[0]} pull [-f] [-1] [-v|-vv]\n')
138 sys
.stderr
.write(f
' YT_CONFIG=... {sys.argv[0]} websub [-1] [-v|-vv]\n')
139 sys
.stderr
.write(f
'-f: force even if still up-to-date-ish\n')
140 sys
.stderr
.write(f
'-v: report errors\n')
141 sys
.stderr
.write(f
'-vv: report accessed feeds\n')
142 sys
.stderr
.write(f
'-1: limit to one feed (for testing it works)\n')
145 verbosity
= 2 if '-vv' in sys
.argv
else 1 if '-v' in sys
.argv
else 0
146 limit
= 1 if '-1' in sys
.argv
else -1
147 force
= '-f' in sys
.argv
149 if 'pull' in sys
.argv
:
150 pull_subscriptions(verbosity
, force
, limit
)
151 elif 'websub' in sys
.argv
:
152 update_subscriptions(verbosity
, limit
)