]> git.gir.st - subscriptionfeed.git/blob - app/common/common.py
common/g_v_i: use parameters from yt-dlp
[subscriptionfeed.git] / app / common / common.py
1 import os
2 import re
3 import json
4 import base64
5 import sqlite3
6 import requests
7 import hmac, hashlib
8 import requests_cache
9 import dateutil.parser
10 from xml.etree import ElementTree
11 from configparser import ConfigParser
12 from datetime import datetime, timezone
13 from urllib.parse import parse_qs, urlparse
14
15 cf = ConfigParser()
16 config_filename = os.environ.get('YT_CONFIG', '/etc/yt/config.ini')
17 cf.read(config_filename)
18 if not 'global' in cf: # todo: full config check
19 raise Exception("Configuration file not found or empty")
20
21 # Note: currently expiring after 10 minutes. googlevideo-urls are valid for 5h59m, but this makes reddit very stale and premiere videos won't start. TODO: exipre when video is livestream/premiere/etc
22 requests_cache.install_cache(backend='memory', expire_after=10*60, allowable_codes=(200,), allowable_methods=('GET', 'HEAD', 'POST'))
23
24 # Note: requests-cache doesn't use redis expiry, so we need this in all backends:
25 # https://github.com/reclosedev/requests-cache/issues/58#issuecomment-164537971
26 # TODO: only run for long-running processes, i.e. the frontend
27 from threading import Timer
28 def purge_cache(sec):
29 requests_cache.remove_expired_responses()
30 t = Timer(sec, purge_cache, args=(sec,))
31 t.setDaemon(True)
32 t.start()
33 purge_cache(10*60)
34
35 # for debugging purposes, monkey patch requests session to store each requests-request in a flask-request's g object (url and response). we can then use a flask error_handler to include the request data in the error log.
36 # since we also call config from outside the flask appcontext, it is wrapped in a try-catch block.
37 from flask import g
38 import requests
39 from requests import Session as OriginalSession
40 class _NSASession(OriginalSession):
41 def request(self, method, url, params=None, data=None, **kwargs):
42 response = super(_NSASession, self).request(
43 method, url, params, data, **kwargs
44 )
45 try:
46 if 'api_requests' not in g:
47 g.api_requests = []
48 g.api_requests.append((url, params, response.text))
49 except RuntimeError: pass # not within flask (e.g. utils.py)
50 return response
51 requests.Session = requests.sessions.Session = _NSASession
52
53 def fetch_xml(feed_type, feed_id):
54 # TODO: handle requests.exceptions.ConnectionError
55 r = requests.get("https://www.youtube.com/feeds/videos.xml", {
56 feed_type: feed_id,
57 })
58 if not r.ok:
59 return None
60
61 return r.content
62
63 def parse_xml(xmldata):
64 ns = {
65 'atom':"http://www.w3.org/2005/Atom",
66 'yt': "http://www.youtube.com/xml/schemas/2015",
67 'media':"http://search.yahoo.com/mrss/",
68 'at': "http://purl.org/atompub/tombstones/1.0",
69 }
70
71 feed = ElementTree.fromstring(xmldata)
72
73 if feed.find('at:deleted-entry',ns):
74 (_,_,vid) = feed.find('at:deleted-entry',ns).get('ref').rpartition(':')
75 return None, None, [{'deleted': True, 'video_id': vid}], None, None
76
77 title = feed.find('atom:title',ns).text
78 author = feed.find('atom:author/atom:name',ns).text \
79 if feed.find('atom:author',ns) else None
80 # for /user/<> endpoint: find out UC-id:
81 # for playlists: this is who created the playlist:
82 try: channel_id = feed.find('yt:channelId',ns).text
83 except:channel_id=None # XXX: why does ternary not work!?
84 # for pullsub: if this exists, we're looking at a playlist:
85 try: playlist_id = feed.find('yt:playlistId',ns).text
86 except:playlist_id=None # XXX: why does ternary not work!?
87 videos = []
88 for entry in feed.findall('atom:entry',ns):
89 videos.append({
90 'video_id': entry.find('yt:videoId',ns).text,
91 'title': entry.find('atom:title',ns).text,
92 'published': entry.find('atom:published',ns).text,
93 'channel_id': entry.find('yt:channelId',ns).text,
94 'author': entry.find('atom:author',ns).find('atom:name',ns).text,
95 # extra fields for pull_subs/webhook:
96 'updated': entry.find('atom:updated',ns).text,
97 })
98
99 return title, author, videos, channel_id, playlist_id
100
101 def update_channel(db, xmldata, from_webhook=False):
102 if not xmldata: return False
103
104 # Note: websub does not return global author, hence taking from first video
105 title, author, videos, channel, playlist = parse_xml(xmldata)
106
107 c = db.cursor()
108 for i, video in enumerate(videos):
109 if video.get('deleted'):
110 # Note: Deletion events are not just fired for actual deletions,
111 # but also for unlisting videos and livestreams that just ended
112 # (even postLiveDVR ones). Hence, we don't follow it.
113 flask_logger(f"ignoring deleted/unlisted/ended video/stream {video['video_id']}")
114 break
115
116 c.execute("SELECT 1 FROM videos WHERE id=?",(video['video_id'],))
117 new_video = len(c.fetchall()) < 1
118 if new_video:
119 # TODO: call store_video_metadata(video_id) here instead and pass video-fallback-metadata to it
120 _, _, meta, _, _ = get_video_info(video['video_id'], metaOnly=True)
121 # The 'published' timestamp sent in websub POSTs are often wrong (e.g.:
122 # video gets uploaded as unlisted on day A and set to public on day B;
123 # the webhook is sent on day B, but 'published' says A. The video
124 # therefore looks like it's just an update to an older video).
125 # g_v_i gives is the date the video was published to viewers, so we
126 # prefer that. But since g_v_i only returns the date without time,
127 # we still use xmlfeed's date if it's the same date.
128 published = dateutil.parser.parse(video['published'])
129 length = None
130 livestream = None
131 premiere = None
132 if meta:
133 meta = video_metadata(meta)
134 published2 = dateutil.parser.parse(meta['published'])
135 if published < published2: # g_v_i date is more accurate:
136 published = published2
137 length = meta['length']
138 livestream = meta['livestream']
139 premiere = meta['premiere']
140
141 now = datetime.now(timezone.utc)
142
143 # we pretend that all videos uploaded this week were uploaded just
144 # now, so the user sees it at the top of the feed, and it doesn't
145 # get inserted somewhere further down.
146 if (now - published).days < 7:
147 timestamp = now
148 else:#, it's just an update to an older video.
149 timestamp = published
150
151 c.execute("""
152 INSERT OR IGNORE INTO videos
153 (id, channel_id, title, length, livestream, premiere, published, crawled)
154 VALUES (?, ?, ?, ?, ?, ?, datetime(?), datetime(?))
155 """, (
156 video['video_id'],
157 video['channel_id'],
158 video['title'],
159 length,
160 livestream,
161 premiere,
162 published,
163 timestamp
164 ))
165 else:
166 # update video title (everything else can't change)
167 c.execute("""
168 UPDATE OR IGNORE videos
169 SET title = ?
170 WHERE id = ?
171 """, (
172 video['title'],
173 video['video_id'],
174 ))
175
176 # for channels, this is obviously always the same, but playlists can
177 # consist of videos from different channels:
178 if i == 0 or playlist:
179 c.execute("""
180 INSERT OR REPLACE INTO channels (id, name)
181 VALUES (?, ?)
182 """, (video['channel_id'], video['author']))
183
184 # keep track of which videos are in a playlist, so we can show the user
185 # why a video is in their feed:
186 if playlist:
187 c.execute("""
188 INSERT OR IGNORE INTO playlist_videos (video_id, playlist_id)
189 VALUES (?, ?)
190 """, (video['video_id'], playlist))
191
192 if playlist and not from_webhook: # Note: playlists can't get updated via websub
193 c.execute("""
194 INSERT OR REPLACE INTO playlists (id, name, author)
195 VALUES (?, ?, ?)
196 """, (playlist, title, channel))
197 c.execute("""
198 INSERT OR REPLACE INTO channels (id, name)
199 VALUES (?, ?)
200 """, (channel, author))
201
202 db.commit()
203
204 return True
205
206 def is_agegated(metadata):
207 playabilityStatus = metadata['playabilityStatus']
208 return bool(
209 playabilityStatus.get("status") == "CONTENT_CHECK_REQUIRED"
210 or playabilityStatus.get("desktopLegacyAgeGateReason")
211 )
212
213 def get_video_info(video_id, *, metaOnly=False, _agegate_bypass=False):
214 """
215 returns: best-quality muxed video stream, stream map, player_response, error-type/mesage
216 error types: player, malformed, livestream, geolocked, agegated, no-url, exhausted
217 """
218 player_error, metadata = None, None # for 'exhausted'
219 with sqlite3.connect(cf['global']['database']) as conn:
220 c = conn.cursor()
221 c.execute("SELECT * FROM captcha_cookies")
222 cookies = dict(c.fetchall())
223 today = datetime.now(timezone.utc).strftime("%Y%m%d")
224 # XXX: anticaptcha hasn't been adapted
225 key = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
226 # ANDROID returns streams that are not throttled or cipher-scambled, but less metadata than WEB.
227 # TVHTML5* returns throttled and possibly ciphered streams, but bypasses age-gate. atm, we don't decipher them.
228 # TODO: unscramble TVHTML5* streams (especially &n= throttling)
229 client = {
230 (False, False): { 'clientName': 'ANDROID', 'clientVersion': '17.31.35', 'androidSdkVersion': 30},
231 (False, True): { 'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER', 'clientVersion': '2.0' },
232 (True, False): { 'clientName': 'WEB', 'clientVersion':f'2.{today}.01.01' },
233 }[(metaOnly, _agegate_bypass)]
234 r = requests.post("https://www.youtube-nocookie.com/youtubei/v1/player", params={'key': key}, json={
235 'videoId': video_id,
236 'context': {
237 'client': {
238 'gl': 'US',
239 'hl': 'en',
240 **client,
241 },
242 'thirdParty': {'embedUrl': 'https://www.youtube.com/'}
243 },
244 }, cookies=cookies, headers={"User-Agent": "com.google.android.youtube/17.31.35 (Linux; U; Android 11) gzip"})
245
246 if not r or r.status_code == 429:
247 return None, None, None, 'banned', 'possible IP ban'
248
249 metadata = r.json()
250 if "error" in metadata:
251 return None, None, metadata, "malformed", metadata.get("error",{}).get("message","")
252 playabilityStatus = metadata['playabilityStatus']['status']
253 if playabilityStatus != "OK":
254 playabilityReason = metadata['playabilityStatus'].get('reason',
255 '//'.join(metadata['playabilityStatus'].get('messages',[])))
256 player_error = f"{playabilityStatus}: {playabilityReason}"
257 if (is_agegated(metadata)
258 and not metaOnly # only need metadata (e.g. called from pubsubhubbub)
259 and not _agegate_bypass
260 ):
261 _, _, metadata_embed, error_embed, errormsg_embed = get_video_info(video_id, _agegate_bypass=True)
262 if error_embed == "player": # agegate bypass failed?
263 return None, None, metadata, 'agegated', player_error
264 elif not error_embed or error_embed in ('livestream','geolocked'):
265 metadata = metadata_embed
266 else:
267 return None, None, metadata, error_embed, errormsg_embed
268 else:
269 # without videoDetails, there's only the error message
270 maybe_metadata = metadata if 'videoDetails' in metadata else None
271 return None, None, maybe_metadata, 'player', player_error
272
273 # livestreams have no adaptive/muxed formats:
274 is_live = metadata['videoDetails'].get('isLive', False)
275
276 if not 'formats' in metadata['streamingData'] and not is_live:
277 return None, None, metadata, 'no-url', player_error
278
279 formats = metadata['streamingData'].get('formats',[])
280 adaptive = metadata['streamingData'].get('adaptiveFormats',[])
281 stream_map = {
282 'adaptive_video': [a for a in adaptive if a['mimeType'].startswith('video/')],
283 'adaptive_audio': [a for a in adaptive if a['mimeType'].startswith('audio/')],
284 'muxed': formats,
285 'hlsManifestUrl': metadata['streamingData'].get('hlsManifestUrl'),
286 }
287
288 try:
289 url = sorted(formats, key=lambda k: k['height'], reverse=True)[0]['url']
290
291 # ip-locked videos can be recovered if the proxy module is loaded:
292 is_geolocked = 'gcr' in parse_qs(urlparse(url).query)
293 except:
294 url = None
295 is_geolocked = False
296
297 nonfatal = 'livestream' if is_live \
298 else 'geolocked' if is_geolocked \
299 else None
300
301 return url, stream_map, metadata, nonfatal, None
302
303 def video_metadata(metadata):
304 if not metadata:
305 return {}
306
307 meta1 = metadata['videoDetails']
308 # With ANDROID player API, we don't get microformat => no publishDate!
309 meta2 = metadata.get('microformat',{}).get('playerMicroformatRenderer',{})
310
311 # sometimes, we receive the notification so early that the length is not
312 # yet populated. Nothing we can do about it.
313 length = int(meta2.get('lengthSeconds',0)) or int(meta1.get('lengthSeconds',0)) or None
314
315 scheduled_time = metadata.get('playabilityStatus',{}) \
316 .get('liveStreamability',{}).get('liveStreamabilityRenderer',{}) \
317 .get('offlineSlate',{}).get('liveStreamOfflineSlateRenderer',{}) \
318 .get('scheduledStartTime')
319 if scheduled_time:
320 scheduled_time = datetime.fromtimestamp(int(scheduled_time)) \
321 .strftime("%Y-%m-%dT%H:%M:%SZ")
322 published_at = (
323 meta2.get('liveBroadcastDetails',{}) .get('startTimestamp') or
324 scheduled_time or
325 f"{meta2.get('publishDate','1970-01-01')}T00:00:00Z"
326 )
327
328 # Note: 'premiere' videos have livestream=False and published= will be the
329 # start of the premiere.
330 return {
331 'title': meta1['title'],
332 'author': meta1['author'],
333 'channel_id': meta1['channelId'],
334 'published': published_at,
335 'views': int(meta1['viewCount']),
336 'length': length,
337 'livestream': meta1['isLiveContent'],
338 'premiere': meta1.get('isUpcoming') and not meta1['isLiveContent'],
339 }
340
341 def store_video_metadata(video_id):
342 # check if we know about it, and if not, fetch and store video metadata
343 with sqlite3.connect(cf['global']['database']) as conn:
344 c = conn.cursor()
345 c.execute("SELECT 1 from videos where id = ?", (video_id,))
346 new_video = len(c.fetchall()) < 1
347 if new_video:
348 _, _, meta, _, _ = get_video_info(video_id, metaOnly=True)
349 if meta:
350 meta = video_metadata(meta)
351 c.execute("""
352 INSERT OR IGNORE INTO videos (id, channel_id, title, length, livestream, premiere, published, crawled)
353 VALUES (?, ?, ?, ?, ?, ?, datetime(?), datetime(?))
354 """, (
355 video_id,
356 meta['channel_id'],
357 meta['title'],
358 meta['length'],
359 meta['livestream'],
360 meta['premiere'],
361 meta['published'],
362 meta['published'],
363 ))
364 c.execute("""
365 INSERT OR REPLACE INTO channels (id, name)
366 VALUES (?, ?)
367 """, (meta['channel_id'], meta['author']))
368
369 def fetch_video_flags(token, video_ids):
370 with sqlite3.connect(cf['global']['database']) as conn:
371 c = conn.cursor()
372 c.execute("""
373 SELECT video_id,display
374 FROM flags
375 WHERE user = ?
376 AND display IS NOT NULL
377 AND video_id IN ({})
378 -- AND display = 'pinned'
379 """.format(",".join(["?"]*len(video_ids))), (token,*video_ids))
380 flags = c.fetchall()
381 pinned = [video for video,disp in flags if disp == 'pinned']
382 hidden = [video for video,disp in flags if disp == 'hidden']
383
384 return pinned, hidden
385
386 from werkzeug.exceptions import NotFound
387 class NoFallbackException(NotFound): pass
388 def fallback_route(*args, **kwargs): # TODO: worthy as a flask-extension?
389 """
390 finds the next route that matches the current url rule, and executes it.
391 args, kwargs: pass all arguments of the current route
392 """
393 from flask import current_app, request, g
394
395 # build a list of endpoints that match the current request's url rule:
396 matching = [
397 rule.endpoint
398 for rule in current_app.url_map.iter_rules()
399 if rule.rule == request.url_rule.rule
400 ]
401 current = matching.index(request.endpoint)
402
403 # since we can't change request.endpoint, we always get the original
404 # endpoint back. so for repeated fall throughs, we use the g object to
405 # increment how often we want to fall through.
406 if not '_fallback_next' in g:
407 g._fallback_next = 0
408 g._fallback_next += 1
409
410 next_ep = current + g._fallback_next
411
412 if next_ep < len(matching):
413 return current_app.view_functions[matching[next_ep]](*args, **kwargs)
414 else:
415 raise NoFallbackException
416
417 def websub_url_hmac(key, feed_id, timestamp, nonce):
418 """ generate sha1 hmac, as required by websub/pubsubhubbub """
419 sig_input = f"{feed_id}:{timestamp}:{nonce}".encode('ascii')
420 return hmac.new(key.encode('ascii'), sig_input, hashlib.sha1).hexdigest()
421
422 def websub_body_hmac(key, body):
423 return hmac.new(key.encode('ascii'), body, hashlib.sha1).hexdigest()
424
425 def flask_logger(msg, level="warning"):
426 level = dict(
427 CRITICAL=50,
428 ERROR=40,
429 WARNING=30,
430 INFO=20,
431 DEBUG=10,
432 NOTSET=0,
433 ).get(level.upper(), 0)
434 try:
435 from flask import current_app
436 current_app.logger.log(level, msg)
437 except:
438 pass
439
440 def pp(*args):
441 from pprint import pprint
442 import sys, codecs
443 pprint(args, stream=codecs.getwriter("utf-8")(sys.stderr.buffer))
Imprint / Impressum