From 4371a55e1161a1604ed22c208c99c92f3b88f5bd Mon Sep 17 00:00:00 2001 From: girst Date: Tue, 9 Mar 2021 19:43:05 +0100 Subject: [PATCH] enable hls proxying, simplify /api/hls_* munging, format code --- app/common/common.py | 31 ++++++++++----------- app/proxy/__init__.py | 42 +++++++++++++---------------- app/youtube/__init__.py | 10 ++++++- app/youtube/templates/watch.html.j2 | 14 ++++++++++ 4 files changed, 58 insertions(+), 39 deletions(-) diff --git a/app/common/common.py b/app/common/common.py index 14c4b8f..f7d48b9 100644 --- a/app/common/common.py +++ b/app/common/common.py @@ -238,12 +238,11 @@ def get_video_info(video_id, sts=0, algo=""): # without videoDetails, there's only the error message maybe_metadata = metadata if 'videoDetails' in metadata else None return None, None, maybe_metadata, 'player', player_error - if metadata['videoDetails'].get('isLive', False): - if sts != -1: # XXX temp for x-hls - return None, None, metadata, 'livestream', None - if not 'formats' in metadata['streamingData']: - if sts != -1: # XXX temp for x-hls + # livestreams have no adaptive/muxed formats: + is_live = metadata['videoDetails'].get('isLive', False) + + if not 'formats' in metadata['streamingData'] and not is_live: continue # no urls formats = metadata['streamingData'].get('formats',[]) @@ -258,20 +257,22 @@ def get_video_info(video_id, sts=0, algo=""): cipher = parse_qs(v.get('cipher') or v.get('signatureCipher')) adaptive[i]['url'] = unscramble(cipher, algo) - stream_map = {'adaptive': adaptive, 'muxed': formats} - stream_map.update({'hlsManifestUrl': metadata['streamingData'].get('hlsManifestUrl')}) - stream_map.update({'dashManifestUrl': metadata['streamingData'].get('dashManifestUrl')}) + stream_map = { + 'adaptive': adaptive, 'muxed': formats, + 'hlsManifestUrl': metadata['streamingData'].get('hlsManifestUrl'), + } - # todo: check if we have urls or try again - if sts != -1: # XXX temp for x-hls - url = sorted(formats, key=lambda k: k['height'], reverse=True)[0]['url'] - else: - url = None + url = sorted(formats, key=lambda k: k['height'], reverse=True)[0]['url'] \ + if not is_live else None # ip-locked videos can be recovered if the proxy module is loaded: - is_geolocked = 'geolocked' if 'gcr' in parse_qs(urlparse(url).query) else None + is_geolocked = 'gcr' in parse_qs(urlparse(url).query) + + nonfatal = 'livestream' if is_live \ + else 'geolocked' if is_geolocked \ + else None - return url, stream_map, metadata, is_geolocked, None + return url, stream_map, metadata, nonfatal, None else: return None, None, metadata, 'exhausted', player_error diff --git a/app/proxy/__init__.py b/app/proxy/__init__.py index 2b3f9ff..6c2ca42 100644 --- a/app/proxy/__init__.py +++ b/app/proxy/__init__.py @@ -24,7 +24,6 @@ def videoplayback(path=None): args = path.split("/")[0:-1] host = dict(zip(args[::2], args[1::2])).get('hls_chunk_host') path = "/" + path.replace(';','%3B').replace('=','%3D') - #host = request.args.get('__host__') else: fvip = request.args.get("fvip", "3") mn = request.args.get("mn", "").split(",")[-1] @@ -52,42 +51,39 @@ def videoplayback(path=None): if k.lower() in REQUEST_HEADERS_WHITELIST } with requests_cache.disabled(): - r = requests.get(f"https://{host}/videoplayback{path}", request.args.to_dict(), headers=req_headers, stream=True) + r = requests.get(f"https://{host}/videoplayback{path}", + request.args.to_dict(), + headers=req_headers, + stream=True) resp_headers = { k:v for k,v in r.headers.items() if k.lower() in RESPONSE_HEADERS_WHITELIST } - response = Response(silence_errors(r.iter_content(chunk_size=8192)), status=r.status_code, headers=resp_headers) + response = Response( + silence_errors(r.iter_content(chunk_size=8192)), + status=r.status_code, headers=resp_headers) response.call_on_close(lambda: r.close()) - response.headers['Access-Control-Allow-Origin'] = '*' + response.headers['Access-Control-Allow-Origin'] = cors_origin() return response @frontend.route("/api/manifest/hls_playlist/") @frontend.route("/api/manifest/hls_variant/") def hls_manifest(path): + path_fixed = request.path.replace(';','%3B').replace('=','%3D') with requests_cache.disabled(): - path_fixed = request.path.replace(';','%3B').replace('=','%3D') - r = requests.get(f"https://manifest.googlevideo.com{path_fixed}", request.args.to_dict()) - if not r.ok: - return "", r.status_code + r = requests.get(f"https://manifest.googlevideo.com{path_fixed}") + if not r.ok: + return "", r.status_code - lines = [] - for line in r.text.splitlines(): - if line.startswith('#'): lines.append(line) - elif line.startswith('https://'): - scheme, _, host, _path = line.split('/', 3) - assert '.googlevideo.com' in host - #lines.append("/" + _path + "?__host__=" + host) - lines.append("/" + _path ) - else: raise StopIteration - #if line.startswith('https://manifest.googlevideo.com/'): - rv = "\n".join(lines) - #rv = r.text - #rv = rv.replace("https://manifest.googlevideo.com/", f"https://{request.host}/") - #rv = re.sub(r"https://([a-z0-9-]+\.googlevideo\.com)", r"https://subscriptions.gir.st/ts-proxy/\1", rv) - return rv, {'content-type': 'application/x-mpegURL', 'Access-Control-Allow-Origin': '*'} + rv = re.sub(r"^https://[a-z0-9-]+\.googlevideo\.com", "", r.text, flags=re.M) + return rv, { + 'Content-Type': 'application/x-mpegURL', + 'Access-Control-Allow-Origin': cors_origin() + } +def cors_origin(): + return f"{request.environ.get('wsgi.url_scheme')}://{request.host}" @frontend.route("/x-hls/") @frontend.route("/x-hls/") diff --git a/app/youtube/__init__.py b/app/youtube/__init__.py index bb11c98..18e52ce 100644 --- a/app/youtube/__init__.py +++ b/app/youtube/__init__.py @@ -84,7 +84,7 @@ def watch(): 'banned': "Instance is being rate limited.", 'malformed': "Video ID is invalid.", 'geolocked': "This video is geolocked.", - 'livestream': "Livestreams not yet supported.", + 'livestream': "Livestreams not supported on this instance.", 'exhausted': errdetails or "Couldn't extract video URLs.", 'player': errdetails, }.get(error) @@ -106,6 +106,14 @@ def watch(): error = None except: pass + try: + if error == 'livestream': + proxy_enabled = url_for('proxy.videoplayback') # will raise if disabled, i think + video_url = urlparse(stream_map['hlsManifestUrl']).path + + error = None + except: pass + # if the instance is blocked, try submitting a job to the anti captcha service: if error == 'banned' and cf['captcha']['api_key']: r2 = requests.get(f'https://www.youtube.com/watch?v={video_id}&hl=en&gl=US') diff --git a/app/youtube/templates/watch.html.j2 b/app/youtube/templates/watch.html.j2 index 8e8e579..339ad7e 100644 --- a/app/youtube/templates/watch.html.j2 +++ b/app/youtube/templates/watch.html.j2 @@ -15,6 +15,9 @@ {% for cc in subtitles %} {% endfor %} +{% if '/api/manifest/hls_variant/' in video_url %}{#todo: this is ugly!#} + +{% endif %} +{% if '/api/manifest/hls_variant/' in video_url %}{#todo: this is ugly!#} + + +{% endif %} {% else %}{#TODO: this'll break livestreams #} -- 2.39.3