From dc001a3a217411d5af1a6f99b2b2617c643e9a18 Mon Sep 17 00:00:00 2001 From: girst Date: Thu, 14 Oct 2021 23:39:44 +0200 Subject: [PATCH] remove now unused cipher fetching code now independent of jwz's youtubedown :^) --- INSTALL.md | 23 +++------- app/common/utils.py | 105 -------------------------------------------- config/setup.sql | 5 --- 3 files changed, 6 insertions(+), 127 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 9560e9b..9b0845c 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -22,13 +22,6 @@ files/init scripts and app/common/utils.py to it. pip3 install -r config/requirements.txt deactivate -## 2.5. Install Perl - -Since we use jwz's `youtubedown`, we need Perl and two external modules. - - dnf install perl perl-IO-Socket-SSL.noarch perl-HTML-Parser.x86_64 - # apt-get install perl libio-socket-ssl-perl libhtml-parser-perl - ## 3. Configuration Configuration is read from the `YT_CONFIG` environment variable, falling back @@ -101,25 +94,21 @@ Cronjobs live in app/common/utils.py, which is both a python script and an executable shell script. When executed with `sh`, it will load the virtual environment and launch itself with it. -Before starting the frontend, run them once. `cipher` is required to play music -videos; `pull` downloads a back catalog of subscriptions; `websub` will -register for automatic push-updated on new videos. (Note: export `YT_CONFIG` if -you don't use `/etc/yt/config`) +Before starting the frontend, run them once. `pull` downloads a back catalog of +subscriptions; `websub` will register for automatic push-updated on new videos. +(Note: export `YT_CONFIG` if you don't use `/etc/yt/config`) ./app/common/utils.py pull ./app/common/utils.py websub - ./app/common/utils.py cipher -If all goes well, install them with `crontab -e`. Running `pull` and `cipher` -once per day is plenty; `websub` should run at least twice daily (both `pull` -and `websub` are idempotent; they won't do any additional work if ran multiple -times). +If all goes well, install them with `crontab -e`. Running `pull` once per day +is plenty; `websub` should run at least twice daily (both `pull` and `websub` +are idempotent; they won't do any additional work if ran multiple times). YT_DIR=/opt/yt # YT_CONFIG=/opt/yt/config/config.ini # optional 19 21 * * * $YT_DIR/app/common/utils.py pull 03 */4 * * * $YT_DIR/app/common/utils.py websub - 52 02 * * * $YT_DIR/app/common/utils.py cipher ## 6. Done diff --git a/app/common/utils.py b/app/common/utils.py index e86e070..8804849 100755 --- a/app/common/utils.py +++ b/app/common/utils.py @@ -137,108 +137,6 @@ def update_feed(feed_id, feed_type, verbose): return True -def refresh_cipher(verbose=1, force=False): - with sqlite3.connect(cf['global']['database']) as conn: - c = conn.cursor() - c.execute("SELECT url FROM cipher") - (player_url,) = c.fetchone() - - new_url = find_player_url(verbose) - if not new_url: - if verbose: - sys.stderr.write(f'FAILED to get player url!\n') - return False - - if player_url == new_url: - if verbose >= 2: - sys.stderr.write(f'player url unchanged.\n') - if not force: - return True - - (cipher_id,) = re.match(r"/s/player/(.*?)\.js", new_url).groups() - sts, algo = ytdown_guess(cipher_id, verbose, force) - if not sts or not algo: - return False - - c.execute(""" - INSERT OR REPLACE INTO cipher (rowid, url, sts, algorithm) - VALUES (0,?,?,?) - """, (new_url, sts, algo)) - - return True - -def find_player_url(verbose, id="jNQXAC9IVRw"): - """ - Extract the player.js URL, which can be passed to youtubedown. - Requests a random video (defaults to the first ever uploaded one, "Me at the - zoo". It shouldn't go away any time soon) and parses the returned Tag Soup. - Note that the URL we're looking for contains our locale, so specify it to - avoid it changing. - """ - class FindPlayer(html.parser.HTMLParser): - def __init__(self, feed): - super().__init__() - self.player_js = None - super().feed(feed) - def handle_starttag(self, tag, attrs): - attrs = dict(attrs) - if tag == "script" and attrs.get("src", "").endswith("/en_US/base.js"): - self.player_js = attrs.get("src") - - if verbose >= 2: - sys.stderr.write(f'fetching embed page {id}\n') - r = requests.get(f"https://www.youtube-nocookie.com/embed/{id}?hl=en&gl=US") - if not r.ok: - if verbose: - sys.stderr.write(f'FAILED {r.status_code}: {r.text[:128]}\n') - return None - - player_js = FindPlayer(r.text).player_js - if verbose >= 2: - sys.stderr.write(f'player.js is {player_js}\n') - return player_js - -def ytdown_guess(cipher_id, verbose, force): - ytdown = "/tmp/youtubedown.pm" - - # update youtubedown once a week: - if force or not os.path.isfile(ytdown) or \ - os.stat(ytdown).st_mtime < time.time()-7*24*60*60 or \ - os.stat(ytdown).st_size == 0: # if previous write failed - if verbose >= 2: - sys.stderr.write('downloading youtubedown\n') - r = requests.get("https://www.jwz.org/hacks/youtubedown") # UA sniffing! - if not r.ok: - if verbose: - sys.stderr.write(f'FAILED {r.status_code}: {r.text[:128]}\n') - return None, None - elif verbose >= 2: - sys.stderr.write(f'done (code {r.status_code})\n') - - with open(ytdown, 'wb') as f: - f.write(r.content) - - perl = subprocess.run( - ["perl", "-wE", """ - require $ARGV[0]; - say guess_cipher($ARGV[1], 0, 0); - """, "--", ytdown, cipher_id - ], - stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE - ) - if perl.returncode > 0: - if verbose: - sys.stderr.write(f'FAILED guess_cipher (exit:{perl.returncode}):\n') - sys.stderr.write(perl.stderr.decode()) - return None, None - sts, algo = perl.stdout.decode('ascii').strip().split(" ", 1) - if verbose >= 2: - sys.stderr.write(f'sts, algo = {sts}, {algo}\n') - return sts, algo - - if __name__ == '__main__': verbosity = 2 if '-vv' in sys.argv else 1 if '-v' in sys.argv else 0 limit = 1 if '-1' in sys.argv else -1 @@ -248,13 +146,10 @@ if __name__ == '__main__': pull_subscriptions(verbosity, force, limit) elif 'websub' in sys.argv: update_subscriptions(verbosity, force, limit) - elif 'cipher' in sys.argv: - refresh_cipher(verbosity, force) else: sys.stderr.write( f'Usage: YT_CONFIG=... {sys.argv[0]} pull [-f] [-1] [-v|-vv]\n' f' YT_CONFIG=... {sys.argv[0]} websub [-f] [-1] [-v|-vv]\n' - f' YT_CONFIG=... {sys.argv[0]} cipher [-f] [-v|-vv]\n' f'-f: force even if still up-to-date-ish\n' f'-v: report errors\n' f'-vv: report accessed feeds\n' diff --git a/config/setup.sql b/config/setup.sql index b3ef48e..5888a15 100644 --- a/config/setup.sql +++ b/config/setup.sql @@ -38,11 +38,6 @@ CREATE TABLE flags( video_id STRING, display TEXT CHECK(display IN (NULL, 'pinned', 'hidden')), PRIMARY KEY (user, video_id)); -CREATE TABLE cipher( - rowid INTEGER PRIMARY KEY CHECK (rowid = 0), -- limit to 1 row - url TEXT, - sts INTEGER, - algorithm TEXT); -- -------------- -- -- REDDIT RELATED -- -- 2.39.3