From e22561fd074186c19c71668624f955fe3b2739fd Mon Sep 17 00:00:00 2001 From: girst Date: Thu, 6 Aug 2020 22:59:14 +0200 Subject: [PATCH] move innertube stuff out of dangerous --- app/common/innertube.py | 91 ++++++++++++++++++++++++++++++++++++- app/dangerous/__init__.py | 94 +-------------------------------------- 2 files changed, 90 insertions(+), 95 deletions(-) diff --git a/app/common/innertube.py b/app/common/innertube.py index de38ce8..fef691c 100644 --- a/app/common/innertube.py +++ b/app/common/innertube.py @@ -26,11 +26,41 @@ def prepare_searchresults(yt_results): def prepare_infocards(metadata): cards = metadata.get('cards',{}).get('cardCollectionRenderer',{}).get('cards',[]) - return [parse_infocard(card) for card in cards if card is not None] + return list(filter(None, map(parse_infocard, cards))) def prepare_endcards(metadata): endsc = metadata.get('endscreen',{}).get('endscreenRenderer',{}).get('elements',[]) - return [parse_endcard(card) for card in endsc if card is not None] + return list(filter(None, map(parse_endcard, endsc))) + +def prepare_channel(result, channel_id): + response = listfind(result,'response') + + meta1 = response.get('metadata',{}).get('channelMetadataRenderer',{}) + meta2 = response.get('microformat',{}).get('microformatDataRenderer',{}) + title = meta1.get('title', meta2.get('title')) + descr = meta1.get('description', meta2.get('description')) # meta2.description is capped at 160chars + thumb = mkthumbs(meta2.get('thumbnail',meta1.get('avatar',{})).get('thumbnails',{})) # .avatar ~ 900px + + if 'continuationContents' in response.keys(): + contents = response['continuationContents'] + try: # TODO: cleanup + items = parse_channel_items(contents['gridContinuation']['items'], channel_id, title) + except: + try: + items = parse_channel_items(contents['sectionListContinuation']['contents'], channel_id, title) + except: + from flask import current_app + current_app.logger.error(result) + items = [] + else: # if absent, we reach end of list + items = [] + + return title, descr, thumb, items + +def prepare_playlist(result): + contents = listfind(result,'response')['continuationContents']['playlistVideoListContinuation'] \ + .get('contents',[]) # no .contents if overran end of playlist + return list(filter(None, map(parse_playlist, contents))) def mkthumbs(thumbs): output = {str(e['height']): e['url'] for e in thumbs} @@ -244,3 +274,60 @@ def parse_endcard(card): else: log_unknown_card(card) return None + +def parse_channel_items(items, channel_id, author): + result = [] + for item in items: + key = next(iter(item.keys()), None) + content = item[key] + if key == "gridVideoRenderer" or key == "videoRenderer": + result.append({'type': 'VIDEO', 'content': { + 'video_id': content['videoId'], + 'title': content['title']['simpleText'], + 'author': author, + 'channel_id': channel_id, + 'length': listfind(content.get('thumbnailOverlays',[]),'thumbnailOverlayTimeStatusRenderer').get('text',{}).get('simpleText'), + 'views': toInt(content.get('viewCountText',{}).get('simpleText')), + 'published': age(content.get('publishedTimeText',{}).get('simpleText')), + }}) + elif key == "gridPlaylistRenderer" or key == "playlistRenderer": + result.append({'type': 'PLAYLIST', 'content': { + 'playlist_id': content['navigationEndpoint']['watchEndpoint']['playlistId'], + 'video_id': content['navigationEndpoint']['watchEndpoint']['videoId'], + 'title': (content['title'].get('simpleText') or # playlistRenderer + content['title']['runs'][0]['text']), # gridPlaylistRenderer + 'author': author, + 'channel_id': channel_id, + 'n_videos': toInt(content.get('videoCount') or # playlistRenderer + content.get('videoCountShortText',{}).get('simpleText') or # grid(1) + content.get('videoCountText',{}).get('runs',[{}])[0].get('text')), # grid(2) + }}) + elif key == "itemSectionRenderer": + result.extend(parse_channel_items(content['contents'], channel_id, author)) + else: + raise Exception(item) # XXX TODO + + return result + +def parse_playlist(item): + key = next(iter(item.keys()), None) + content = item[key] + if key == "playlistVideoRenderer": + if not content.get('isPlayable', False): + return None # private or deleted video + + return {'type': 'VIDEO', 'content': { + 'video_id': content['videoId'], + 'title': (content['title'].get('simpleText') or # playable videos + content['title'].get('runs',[{}])[0].get('text')), # "[Private video]" + 'playlist_id': content['navigationEndpoint']['watchEndpoint']['playlistId'], + 'index': content['navigationEndpoint']['watchEndpoint']['index'], #or int(content['index']['simpleText']) + # rest is missing from unplayable videos: + 'author': content.get('shortBylineText',{}).get('runs',[{}])[0].get('text'), + 'channel_id':content.get('shortBylineText',{}).get('runs',[{}])[0].get('navigationEndpoint',{}).get('browseEndpoint',{}).get('browseId'), + 'length': (content.get("lengthText",{}).get("simpleText") or # "8:51" + int(content.get("lengthSeconds", 0))), # "531" + 'starttime': content['navigationEndpoint']['watchEndpoint'].get('startTimeSeconds'), + }} + else: + raise Exception(item) # XXX TODO diff --git a/app/dangerous/__init__.py b/app/dangerous/__init__.py index 37775b0..a2a0626 100644 --- a/app/dangerous/__init__.py +++ b/app/dangerous/__init__.py @@ -55,7 +55,7 @@ def channel(channel_id, subpage="videos"): result = fetch_ajax(make_channel_params(channel_id, subpage, page, sort_by)) - title, descr, thumb, rows = prepare_channel_items(result, channel_id) + title, descr, thumb, rows = prepare_channel(result, channel_id) # TODO: add is_pinned/is_hidden return render_template('channel.html.j2', @@ -67,98 +67,6 @@ def channel(channel_id, subpage="videos"): channel_desc=descr, page=page) -############ - -# TODO: this belongs in common.innertube -def prepare_channel_items(result, channel_id): - response = listfind(result,'response') - - meta1 = response.get('metadata',{}).get('channelMetadataRenderer',{}) - meta2 = response.get('microformat',{}).get('microformatDataRenderer',{}) - title = meta1.get('title', meta2.get('title')) - descr = meta1.get('description', meta2.get('description')) # meta2.description is capped at 160chars - thumb = mkthumbs(meta2.get('thumbnail',meta1.get('avatar',{})).get('thumbnails',{})) # .avatar ~ 900px - - if 'continuationContents' in response.keys(): - contents = response['continuationContents'] - try: # TODO: cleanup - items = parse_channel_items(contents['gridContinuation']['items'], channel_id, title) - except: - try: - items = parse_channel_items(contents['sectionListContinuation']['contents'], channel_id, title) - except: - from flask import current_app - current_app.logger.error(result) - items = [] - else: # if absent, we reach end of list - items = [] - - return title, descr, thumb, items - -def parse_channel_items(items, channel_id, author): - result = [] - for item in items: - key = next(iter(item.keys()), None) - content = item[key] - if key == "gridVideoRenderer" or key == "videoRenderer": - result.append({'type': 'VIDEO', 'content': { - 'video_id': content['videoId'], - 'title': content['title']['simpleText'], - 'author': author, - 'channel_id': channel_id, - 'length': listfind(content.get('thumbnailOverlays',[]),'thumbnailOverlayTimeStatusRenderer').get('text',{}).get('simpleText'), - 'views': toInt(content.get('viewCountText',{}).get('simpleText')), - 'published': age(content.get('publishedTimeText',{}).get('simpleText')), - }}) - elif key == "gridPlaylistRenderer" or key == "playlistRenderer": - result.append({'type': 'PLAYLIST', 'content': { - 'playlist_id': content['navigationEndpoint']['watchEndpoint']['playlistId'], - 'video_id': content['navigationEndpoint']['watchEndpoint']['videoId'], - 'title': (content['title'].get('simpleText') or # playlistRenderer - content['title']['runs'][0]['text']), # gridPlaylistRenderer - 'author': author, - 'channel_id': channel_id, - 'n_videos': toInt(content.get('videoCount') or # playlistRenderer - content.get('videoCountShortText',{}).get('simpleText') or # grid(1) - content.get('videoCountText',{}).get('runs',[{}])[0].get('text')), # grid(2) - }}) - elif key == "itemSectionRenderer": - result.extend(parse_channel_items(content['contents'], channel_id, author)) - else: - raise Exception(item) # XXX TODO - - return result - -def prepare_playlist(result): - contents = listfind(result,'response')['continuationContents']['playlistVideoListContinuation'] \ - .get('contents',[]) # no .contents if overran end of playlist - return list(filter(None, map(parse_playlist, contents))) - -def parse_playlist(item): - key = next(iter(item.keys()), None) - content = item[key] - if key == "playlistVideoRenderer": - if not content.get('isPlayable', False): - return None # private or deleted video - - return {'type': 'VIDEO', 'content': { - 'video_id': content['videoId'], - 'title': (content['title'].get('simpleText') or # playable videos - content['title'].get('runs',[{}])[0].get('text')), # "[Private video]" - 'playlist_id': content['navigationEndpoint']['watchEndpoint']['playlistId'], - 'index': content['navigationEndpoint']['watchEndpoint']['index'], #or int(content['index']['simpleText']) - # rest is missing from unplayable videos: - 'author': content.get('shortBylineText',{}).get('runs',[{}])[0].get('text'), - 'channel_id':content.get('shortBylineText',{}).get('runs',[{}])[0].get('navigationEndpoint',{}).get('browseEndpoint',{}).get('browseId'), - 'length': (content.get("lengthText",{}).get("simpleText") or # "8:51" - int(content.get("lengthSeconds", 0))), # "531" - 'starttime': content['navigationEndpoint']['watchEndpoint'].get('startTimeSeconds'), - }} - else: - raise Exception(item) # XXX TODO - -############ - @frontend.route('/playlist') def playlist(): #TODO: if anything goes wrong, fall back to xmlfeed -- 2.39.3