From 99292a12018d915c5ff1c9bb85be4881230d9091 Mon Sep 17 00:00:00 2001 From: girst Date: Sun, 21 Jan 2024 14:53:36 +0000 Subject: [PATCH] support (hiding) youtube shorts everywhere proper support for youtube shorts in search results, channel pages, playlists, homepage/subscriptions - especially hiding them. --- app/browse/__init__.py | 9 ++++++--- app/browse/innertube.py | 26 +++++++++++++++----------- app/common/common.py | 9 +++++++-- app/common/user.py | 15 +++++++++++++++ app/youtube/__init__.py | 14 ++------------ 5 files changed, 45 insertions(+), 28 deletions(-) diff --git a/app/browse/__init__.py b/app/browse/__init__.py index 6330f99..365ac59 100644 --- a/app/browse/__init__.py +++ b/app/browse/__init__.py @@ -18,6 +18,7 @@ frontend = Blueprint('browse', __name__, @frontend.route('/search') def search(): token = getattr(current_user, 'token', 'guest') + settings = getattr(current_user, 'get_settings', lambda: {})() q = request.args.get('q') or request.args.get('search_query') continuation = request.args.get('continuation') @@ -38,7 +39,7 @@ def search(): )) results, extras, continuation = prepare_searchresults(yt_results) - results = apply_video_flags(token, results) + results = apply_video_flags(token, results, settings) for extra in extras: flash(extra, 'info') @@ -51,6 +52,7 @@ def search(): @frontend.route('/channel//') def channel(channel_id, subpage="videos"): token = getattr(current_user, 'token', 'guest') + settings = getattr(current_user, 'get_settings', lambda: {})() if subpage in ("videos", "streams", "shorts"): # "streams"==livestreams sort_by = request.args.get('sort') or "newest" query = None @@ -92,7 +94,7 @@ def channel(channel_id, subpage="videos"): return fallback_route(channel_id, subpage) # set pin/hide stati of retrieved videos: - rows = apply_video_flags(token, rows) + rows = apply_video_flags(token, rows, settings) with sqlite3.connect(cf['global']['database']) as conn: c = conn.cursor() @@ -142,6 +144,7 @@ def channel_redirect(user, subpage=None): @frontend.route('/playlist') def playlist(): token = getattr(current_user, 'token', 'guest') + settings = getattr(current_user, 'get_settings', lambda: {})() playlist_id = request.args.get('list') if not playlist_id: raise BadRequest("No playlist ID") @@ -161,7 +164,7 @@ def playlist(): return fallback_route() title, author, channel_id, rows, continuation = prepare_playlist(result) - rows = apply_video_flags(token, rows) + rows = apply_video_flags(token, rows, settings) return render_template('playlist.html.j2', title=title, diff --git a/app/browse/innertube.py b/app/browse/innertube.py index 176f3eb..6c4e787 100644 --- a/app/browse/innertube.py +++ b/app/browse/innertube.py @@ -85,8 +85,8 @@ def prepare_channel(response, channel_id, channel_name): def prepare_playlist(result): contents = result['continuationContents'] - unparsed = contents['playlistVideoListContinuation'].get('contents',[]) - more = ( + unparsed = contents|G('playlistVideoListContinuation', 'richGridContinuation')|G('contents') or [] + more = ( # XXX: unavailable if richGridContinuation contents |G('playlistVideoListContinuation') |G('continuations') @@ -149,6 +149,7 @@ def parse_result_items(items): 'views': content|G('viewCountText')|G.text|A.int or 0, # "1,234 {views|watching}", absent on 0 views 'published': content|G('publishedTimeText')|G('simpleText')|A(age), 'live': content|G('badges')|Select('metadataBadgeRenderer')|G('style')=='BADGE_STYLE_TYPE_LIVE_NOW', + 'shorts': key == 'reelItemRenderer', }}) elif key in ['playlistRenderer', 'radioRenderer', 'showRenderer']: # radio == "Mix" playlist, show == normal playlist, specially displayed results.append({'type': 'PLAYLIST', 'content': { @@ -221,7 +222,7 @@ def parse_channel_items(items, channel_id, author): for item in items: key = next(iter(item.keys()), None) content = item[key] - if key in ["gridVideoRenderer", "videoRenderer", "videoCardRenderer", 'reelItemRenderer']: # reel==youtube-shorts + if key in ["gridVideoRenderer", "videoRenderer", "videoCardRenderer", 'reelItemRenderer']: # only videoCardRenderer (topic channels) has author and channel, others fall back to supplied ones. result.append({'type': 'VIDEO', 'content': { 'video_id': content['videoId'], @@ -237,6 +238,7 @@ def parse_channel_items(items, channel_id, author): # topic channel: .metadataText.simpleText = "22M views \u00b7 2 months ago" 'views': content|G('viewCountText')|G.text|A.int, 'published': content|G('publishedTimeText')|G.text|A(age), + 'shorts': key == 'reelItemRenderer', }}) elif key in ["gridPlaylistRenderer", "playlistRenderer", "gridRadioRenderer"]: result.append({'type': 'PLAYLIST', 'content': { @@ -304,22 +306,24 @@ def parse_channel_items(items, channel_id, author): def parse_playlist(item): key = next(iter(item.keys()), None) content = item[key] - if key == "playlistVideoRenderer": - if not content.get('isPlayable', False): + if key in ["playlistVideoRenderer", "reelItemRenderer"]: + if not content.get('isPlayable', False) and key!="reelItemRenderer": return None # private or deleted video return {'type': 'VIDEO', 'content': { 'video_id': content['videoId'], - 'title': (content['title'].get('simpleText') or # playable videos - content['title'].get('runs',[{}])[0].get('text')), # "[Private video]" - 'playlist_id': content['navigationEndpoint']['watchEndpoint']['playlistId'], - 'index': content['navigationEndpoint']['watchEndpoint'].get('index',0), #or int(content['index']['simpleText']) (absent on course intros; e.g. PL96C35uN7xGJu6skU4TBYrIWxggkZBrF5) - # rest is missing from unplayable videos: + 'title': content|G('title', 'headline')|G.text, + 'playlist_id': content['navigationEndpoint']|G('watchEndpoint', 'reelWatchEndpoint')|G('playlistId'), + 'index': content['navigationEndpoint'].get('watchEndpoint',{}).get('index',0), #or int(content['index']['simpleText']) (absent on course intros; e.g. PL96C35uN7xGJu6skU4TBYrIWxggkZBrF5 or on shorts; e.g. PLnN2bBxGARv7fRxsCcWaxvGE6sn5Ypp1H) + # rest is missing from unplayable videos and from shorts: 'author': content.get('shortBylineText',{}).get('runs',[{}])[0].get('text'), 'channel_id':content.get('shortBylineText',{}).get('runs',[{}])[0].get('navigationEndpoint',{}).get('browseEndpoint',{}).get('browseId'), 'length': (content.get("lengthText",{}).get("simpleText") or # "8:51" int(content.get("lengthSeconds", 0))), # "531" - 'starttime': content['navigationEndpoint']['watchEndpoint'].get('startTimeSeconds'), + 'starttime': content['navigationEndpoint'].get('watchEndpoint',{}).get('startTimeSeconds'), + 'shorts': key=="reelItemRenderer" }} + elif key == "richItemRenderer": + return parse_playlist(content['content']) # should contain one ytshorts else: raise Exception(item) # XXX TODO diff --git a/app/common/common.py b/app/common/common.py index 3a58802..9532f55 100644 --- a/app/common/common.py +++ b/app/common/common.py @@ -454,13 +454,18 @@ def fetch_video_flags(token, video_ids): return pinned, hidden -def apply_video_flags(token, rows): +def apply_video_flags(token, rows, settings={}): video_ids = [card['content']['video_id'] for card in rows if 'video_id' in card['content']] pinned, hidden = fetch_video_flags(token, video_ids) + noshorts = settings.get('noshorts') or False return sorted([ {'type':v['type'], 'content':{**v['content'], 'pinned': v['content']['video_id'] in pinned if 'video_id' in v['content'] else False}} for v in rows - if 'video_id' not in v['content'] or v['content']['video_id'] not in hidden + if ( + 'video_id' not in v['content'] or v['content']['video_id'] not in hidden + ) and ( + not (noshorts and v['content'].get('shorts')) + ) ], key=lambda v:v['content']['pinned'], reverse=True) from werkzeug.exceptions import NotFound diff --git a/app/common/user.py b/app/common/user.py index d935bf0..13aba5c 100644 --- a/app/common/user.py +++ b/app/common/user.py @@ -22,6 +22,21 @@ class User(UserMixin): # TODO: to common c.execute("UPDATE users SET password = ? where id = ?", (self.passwd, self.id,)) def check_password(self, passwd): return check_password_hash(self.passwd, passwd) + def get_settings(self): + settings = {} # fallback for guest user + if self.is_authenticated: + with sqlite3.connect(cf['global']['database']) as conn: + c = conn.cursor() + c.execute(""" + SELECT setting, value + FROM user_settings + WHERE user_id = ? + """, (self.id,)) + settings = { + setting: json.loads(value) + for setting, value in c.fetchall() + } + return settings @classmethod def from_id(self, id): with sqlite3.connect(cf['global']['database']) as conn: diff --git a/app/youtube/__init__.py b/app/youtube/__init__.py index 1ddfe60..0f8aa27 100644 --- a/app/youtube/__init__.py +++ b/app/youtube/__init__.py @@ -26,26 +26,16 @@ def index(): def feed(): if current_user.is_anonymous: token = 'guest' + settings = {} if 'welcome_message' in cf['frontend']: flash(cf['frontend']['welcome_message'], "welcome") else: token = current_user.token + settings = current_user.get_settings() page = request.args.get('page', 0, type=int) with sqlite3.connect(cf['global']['database']) as conn: c = conn.cursor() - settings = {} # fallback for guest user - if current_user.is_authenticated: - c.execute(""" - SELECT setting, value - FROM user_settings - WHERE user_id = ? - """, (current_user.id,)) - settings = { - setting: json.loads(value) - for setting, value in c.fetchall() - } - c.execute(""" SELECT videos.id, channel_id, name, title, length, livestream, premiere, shorts, published > datetime('now') as upcoming, published, playlist_videos.playlist_id, display FROM videos -- 2.39.3