From 4c3c6e3db4ccab6c41e12ebb1baf644971fe4dd9 Mon Sep 17 00:00:00 2001 From: girst Date: Tue, 2 Jun 2020 17:41:13 +0200 Subject: [PATCH] switch to html watch page by default, fix csrf bugs --- INSTALL | 3 +++ README.md | 7 +++--- app/frontend.py | 31 +++++++++++++++---------- app/templates/macros.imp.j2 | 21 +++++++++++------ app/templates/watch.html.j2 | 45 +++++++++++++++++++++++-------------- 5 files changed, 67 insertions(+), 40 deletions(-) diff --git a/INSTALL b/INSTALL index e5ac2b6..2fa4646 100644 --- a/INSTALL +++ b/INSTALL @@ -13,6 +13,9 @@ gunicorn-frontend-config.py -> https://docs.gunicorn.org/en/stable/settings.html gunicorn-webhooks-config.py startup.sh +4a. generate a secret key in app/frontend.py + # replace 'app.secret_key = ...' with the output of this command: + python3 -c 'import secrets;print(secrets.token_bytes(16))' 5. for now, edit pull-subs and update-subs to point to your venv 6. manually trigger cronjobs ./app/refresh-cipher.pl diff --git a/README.md b/README.md index 01f6f0f..af96a45 100644 --- a/README.md +++ b/README.md @@ -9,19 +9,18 @@ Test Instance: http://delta.gir.st:8000/ (will move in the future) TODO: - implement authentication - abstract database access, implement nice config system - - fetch cipher.txt and cache it in memory - - use 'card'-css for Info- and Endcards - don't hardcode reddit-links in the template - use html error pages for watch?show=metadata - use invidious api for channels/playlists/search (with query parameter provider=https://invidio.us/api or similar) - rewrite frontend-'card' with flexbox https://css-tricks.com/snippets/css/a-guide-to-flexbox/ - write documentation, theory of operation, overview diagram, risk (of getting banned) assessment, ... - - provide gunicorn configs, config.ini, ... - - organize repo directory structure, automate install/setup somewhat - asynchronically call update-subs and pull-subs after subscribing to a channel - we are currently misclassifying some subscription videos as old, when they are uploaded unlisted and made public later. this could be solved by querying get_video_info for websub-videos that are not yet in the database to get the actual published date. +wishlist: + - proxy googlevideo (and probably thumnails) responses + # Installation see INSTALL file diff --git a/app/frontend.py b/app/frontend.py index 4e36e4c..cabf302 100644 --- a/app/frontend.py +++ b/app/frontend.py @@ -62,22 +62,24 @@ def watch(): if not 'v' in request.args: return "missing video id", 400 + plaintextheader = {'content-type': 'text/plain',"Link": "; rel=stylesheet;"} + video_id = request.args.get('v') (video_url, metadata, error_type, error) = get_video_info(video_id) if error_type in ['initial', 'player']: - return error, 400, {'content-type': 'text/plain',"Link": "; rel=stylesheet;"} + return error, 400, plaintextheader show = request.args.get("show") - if show == "metadata": # todo: handle the case when we have an exhausted error with no metadata returned - return render_template('watch.html.j2', video_id=video_id, video_url=video_url, **prepare_metadata(metadata)) - elif show == "json": - return jsonify(metadata) - else: + if show == "raw": if error: extra = {'geolocked':'local=1', 'livestream':'raw=0'}.get(error,'') # if error==exhausted, metadata.playabilityStatus.reason may contain additional information. - return f"{error.upper()}: Redirecting to Invidious.", 502, {'Refresh': '2; URL=https://invidio.us/watch?v='+video_id+'&'+extra+'&raw=1','content-type': 'text/plain',"Link": "; rel=stylesheet;"} + return f"{error.upper()}: Redirecting to Invidious.", 502, {'Refresh': f'2; URL=https://invidio.us/watch?v={video_id}&{extra}&raw=1', **plaintextheader} return redirect(video_url, code=307) + elif show == "json": + return jsonify(metadata) + else: # todo: handle geolocked, livesteam and the case when we have an exhausted error with no metadata returned + return render_template('watch.html.j2', video_id=video_id, video_url=video_url, **prepare_metadata(metadata)) def prepare_metadata(metadata): meta1 = metadata['videoDetails'] @@ -120,15 +122,16 @@ def prepare_metadata(metadata): ctype = "PLAYLIST" content = { 'playlist_id': content['action']['watchEndpoint']['playlistId'], - 'video_id': content['action']['watchEndpoint']['videoId'], # XXX: untested + 'video_id': content['action']['watchEndpoint']['videoId'], 'title': content['playlistTitle']['simpleText'], 'author': content['channelName']['simpleText'], - 'n_videos': content['videoCountText']['simpleText'], + 'n_videos': content['playlistVideoCount']['simpleText'], # '21' } elif ctype == "simpleCardContentRenderer" and 'urlEndpoint' in content.get('command',{}).keys(): ctype = "WEBSITE" content = { 'url': parse_qs(content['command']['urlEndpoint']['url'].split('?')[1])['q'][0], + 'domain': content['displayDomain']['simpleText'], 'title': content['title']['simpleText'], 'text': content['actionButton']['simpleCardButtonRenderer']['text']['simpleText'], } @@ -160,11 +163,12 @@ def prepare_metadata(metadata): 'video_id': card['endpoint']['watchEndpoint']['videoId'], 'title': card['title']['simpleText'], 'author': card['metadata']['simpleText'], - 'n_videos': card['playlistLength']['simpleText'], + 'n_videos': card['playlistLength']['simpleText'].replace(" videos", ""), } elif ctype == "WEBSITE": content = { 'url': parse_qs(card['endpoint']['urlEndpoint']['url'].split('?')[1])['q'][0], + 'domain': card['metadata']['simpleText'], 'title': card['title']['simpleText'], 'icons': {e['height']: e['url'] for e in card['image']['thumbnails']}, } @@ -187,6 +191,7 @@ def prepare_metadata(metadata): 'aspectr': aspect_ratio, 'unlisted': meta2['isUnlisted'], 'countries': meta2['availableCountries'], + 'poster': meta2['thumbnail']['thumbnails'][0]['url'], 'infocards': [parse_infocard(card) for card in cards], 'endcards': [parse_endcard(card) for card in endsc], 'subtitles': subtitles, @@ -302,7 +307,7 @@ def subscription_manager(): def feed_post(): token = request.args.get('token', 'guest') if token == 'guest': return "guest user is read-only", 403 - action = next(request.form.keys(), None) + action = next(iter(k for k in request.form.keys() if k != 'csrf'), None) if action in ['pin', 'unpin', 'hide']: video_id = request.form.get(action) display = { @@ -425,7 +430,7 @@ def add_csrf_protection(response): if response.mimetype == "text/html": token = hmac.new(app.secret_key, request.remote_addr.encode('ascii'), hashlib.sha256).hexdigest() # TODO: will fail behind reverse proxy (remote_addr always localhost) response.set_data( re.sub( - rb'()', # match form tags with any number of attributes and any type of quotes + rb'''(<[Ff][Oo][Rr][Mm](\s+[a-zA-Z0-9-]+(=(\w*|'[^']*'|"[^"]*"))?)*>)''', # match form tags with any number of attributes and any type of quotes rb'\1', # hackily append a hidden input with our csrf protection value response.get_data())) return response @@ -434,6 +439,8 @@ def verify_csrf_protection(): token = hmac.new(app.secret_key, request.remote_addr.encode('ascii'), hashlib.sha256).hexdigest() # TODO: will fail behind reverse proxy (remote_addr always localhost) if request.method == "POST" and request.form.get('csrf') != token: return "CSRF validation failed!", 400 + request.form = request.form.copy() # make it mutable + # request.form.pop('csrf') # XXX: breaks all requests?! @app.template_filter('format_date') def format_date(s): diff --git a/app/templates/macros.imp.j2 b/app/templates/macros.imp.j2 index 6f1f491..4cc06d1 100644 --- a/app/templates/macros.imp.j2 +++ b/app/templates/macros.imp.j2 @@ -1,10 +1,10 @@ -{% macro card(video_id, title='') -%}{# +{% macro card_generic(link, thumbnail, title='') -%}{# #}
- - + +
- {{ title | e }} + {{ title | e }}
{{ caller() }} @@ -12,20 +12,27 @@
{# #}{%- endmacro %} +{% macro card(video_id, title='') -%}{# + #}{% set caller_ = caller %}{# + #}{% call card_generic("/watch?v="~video_id, "https://i.ytimg.com/vi/"~video_id~"/mqdefault.jpg", title) %} + {{ caller_() }} + {% endcall %}{# +#}{%- endmacro %} + {% macro infobar_subscriptions(video_id, channel_id, author, published, pinned="undefined") -%} {{ author | e }}
{{ published }} {% if pinned != "undefined" %}
- +
- +
{%endif%} - 💡 + 🎞️
{%- endmacro %} {% macro infobar_reddit(comments, karma) -%} diff --git a/app/templates/watch.html.j2 b/app/templates/watch.html.j2 index 9659c44..9301c8b 100644 --- a/app/templates/watch.html.j2 +++ b/app/templates/watch.html.j2 @@ -3,10 +3,12 @@ +{% import 'macros.imp.j2' as macros %} +
-
-- 2.39.3