From db42e548cff62c4c2b0b45cbb60b06cd7c55c1a2 Mon Sep 17 00:00:00 2001 From: girst Date: Fri, 15 Oct 2021 03:39:41 +0200 Subject: [PATCH] provide a semi-automatic install mechanism --- INSTALL.md | 116 +++++++++-------------------- Makefile | 33 ++++++++ config/create-admin.sh | 6 ++ config/generate-passwd.py | 16 ---- config/gunicorn-frontend-config.py | 24 +++--- 5 files changed, 85 insertions(+), 110 deletions(-) create mode 100644 Makefile create mode 100755 config/create-admin.sh delete mode 100755 config/generate-passwd.py diff --git a/INSTALL.md b/INSTALL.md index 9b0845c..4d58fb1 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,74 +1,37 @@ # Installation Instructions -Note that these instructions are a work in progress and will change during the prototyping phase. -## 1. Download the Project +## Simple Install -Grab and extract a tarball or clone this repository into a convenient location, -e.g. `/opt/yt` (the rest of these instructions will assume this path). +Execute `sudo make install`. This will do the following: +- create a python virtual environment in `/opt/yt/venv` +- install the software in `/opt/yt/app` +- copy config files to `/etc/yt` - git clone ... /opt/yt - cd /opt/yt +You are then expected to edit `/etc/yt/config.ini` and `/etc/yt/gunicorn-frontend-config.py`. +Finally, run `sudo make finish` to enable the systemd units and start populating the guest feed. -## 2. Install Virtual Environment -Minimal required python version is 3.6. +## Advanced: customizing the installation -For the most part, it doesn't matter where you set up the virtual environment. -Here, it is created in `/opt/yt/venv`. You eill have to point the systemd unit -files/init scripts and app/common/utils.py to it. +You should only read this section if you cannot install using `sudo make install`. - python3 -m venv venv - . venv/bin/activate - pip3 install -r config/requirements.txt - deactivate +The systemd unit files can be modified to install the software, virtualenv +and/or config files into different locations. +- `WorkingDirectory` should point to the parent directory of `app/`; +- `Environment=PATH` to the virtualenv's `bin/`; +- `Environment=YT_CONFIG` to the `config.ini`. +- The gunicorn config can be set by modifying `ExecStart`. -## 3. Configuration +To allow using classic cronjobs to run the websub/rss jobs, `utils.py` is a +polyglot file that when executed re-launches itself in the virtual environment. +For this to work, the venv must be placed in `venv/` next to the `app/` +directory. -Configuration is read from the `YT_CONFIG` environment variable, falling back -to `/etc/yt/config.ini`. You can keep the `config/` directory untouched to -avoid merge conflicts, and copy the relevant files to `/etc/yt`. - -Carefully read and fill out `config/config.ini`. If you want to run the app -directly from gunicorn (as I do), the [Gunicorn configuration] file -(`gunicorn-frontend-config.py`) will need TLS certificate paths. You can ignore -`gunicorn-webhooks-config.py`, unless you want to run it standalone. Otherwise, -configure them to use a high port and reverse-proxy them through your real web -server. -`gunicorn-frontend-config.py` uses eventlet workers, which require -`gunicorn[eventlet]` to be installed. If you want a synchronous worker, use -`gthread` instead. If you are using the in-memory requests-cache, do not use -more than 1 process, or the cache will be useless! - -If you want to use the included systemd unit files -(`subscriptions-frontend.service` and optionally -`subscriptions-webhooks.service`), point `WorkingDirectory=` to the location of -the repository, `Environment=PATH=` to the location of the virtualenv's `bin/` -directory and `Environment=YT_CONFIG=` to your `config.ini` (if you aren't -using `/etc/yt/config.ini`), as well as the gunicorn config path. Then copy -them to `/etc/systemd/system/`. -A HTTP-to-HTTPS redirect "server" is available as -`subscriptions-port80.service` (requires netcat(1)); it needs the domain name -of your instance in `Environment=DOMAIN=`. - -**Do not start the frontend before the database and cronjobs are in place!** - - cp -r config /etc/yt - cp config/subscriptions-frontend.service /etc/systemd/system - - vi /etc/yt/config.ini - vi /etc/yt/gunicorn-frontend-config.py - vi /etc/systemd/system/subscriptions-frontend.service - - systemctl daemon-reload - systemctl enable subscriptions-frontend.service - -[Gunicorn configuration]: https://docs.gunicorn.org/en/stable/settings.html - -## 4. Create and Prepopulate SQLite Database +**Do not start the frontend/timers/cronjobs before the database is in place!** To create the database and tables, simply issue: - sqlite3 subscriptions.sqlite < config/setup.sql + sqlite3 subscriptions.sqlite < config/setup.sql Next, create a user for yourself: @@ -88,30 +51,19 @@ can use the provided list: You can do the same for your own subscriptions, or use the *Subscription Manager* in the footer of the main page to subscribe to channels manually. -## 5. Set up Cron Jobs - -Cronjobs live in app/common/utils.py, which is both a python script and an -executable shell script. When executed with `sh`, it will load the virtual -environment and launch itself with it. - -Before starting the frontend, run them once. `pull` downloads a back catalog of -subscriptions; `websub` will register for automatic push-updated on new videos. -(Note: export `YT_CONFIG` if you don't use `/etc/yt/config`) - - ./app/common/utils.py pull - ./app/common/utils.py websub - -If all goes well, install them with `crontab -e`. Running `pull` once per day -is plenty; `websub` should run at least twice daily (both `pull` and `websub` -are idempotent; they won't do any additional work if ran multiple times). - - YT_DIR=/opt/yt - # YT_CONFIG=/opt/yt/config/config.ini # optional - 19 21 * * * $YT_DIR/app/common/utils.py pull - 03 */4 * * * $YT_DIR/app/common/utils.py websub +Before starting the frontend, you should run the cronjobs/timers once. +`update.py pull` downloads a back catalog of subscriptions; `update.py websub` +will register for automatic push-updated on new videos. Note that this will +take a long while, as we only do 1 request per minute to Google's RSS endpoints. -## 6. Done + systemctl start subscriptions-update@pull.timer + systemctl start subscriptions-update@websub.timer -All set -- you can start the frontend now and relax. +If you don't want to use a reverse-proxy, [gunicorn can be configured] to +directly serv traffic over TLS by setting a certificate (`certfile`) and +private key (`keyfile`). However, it won't listen on http/tcp to redirect +traffic to https/tcp. For this, a very basic HTTP-to-HTTPS redirect "server" is +available as `subscriptions-port80.service` (requires netcat(1)); it needs the +domain name of your instance in `Environment=DOMAIN=`. - systemctl start subscriptions-frontend.service +[gunicorn can be configured]: https://docs.gunicorn.org/en/stable/settings.html diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..f04e11b --- /dev/null +++ b/Makefile @@ -0,0 +1,33 @@ +.PHONY: all install config + +all: + @echo 'issue "sudo make install" to install subscriptionfeed' + +install: + install -d /opt/yt + cp -a app /opt/yt + install -Dt /etc/yt config/config.ini config/gunicorn-frontend-config.py + install -Dt /etc/systemd/system config/subscriptions-frontend.service + install -Dt /etc/systemd/system config/subscriptions-port80.service + install -Dt /etc/systemd/system config/subscriptions-update@.service + install -Dt /etc/systemd/system config/subscriptions-update@.timer + systemctl daemon-reload + python3 -m venv /opt/yt/venv + /opt/yt/venv/bin/pip install -r config/requirements.txt + sqlite3 /opt/yt/subscriptions.sqlite < config/setup.sql + sqlite3 /opt/yt/subscriptions.sqlite < config/guest.sql + # admin user with default password -- must be changed on first login! + ./config/create-admin.sh /opt/yt/subscriptions.sqlite + @echo '###################################################################' + @echo '# installation finished! #' + @echo '# edit /etc/yt/config.ini and /etc/yt/gunicorn-frontend-config.py #' + @echo '# then run "sudo make finish" to start the server #' + @echo '###################################################################' + +finish: + systemctl enable subscriptions-frontend.service + systemctl enable subscriptions-update@websub.timer + systemctl enable subscriptions-update@pull.timer + systemctl start subscriptions-frontend.service + # one-shot to initialize db with guest subscriptions in the background: + systemctl start subscriptions-update@pull.service & diff --git a/config/create-admin.sh b/config/create-admin.sh new file mode 100755 index 0000000..803a0f4 --- /dev/null +++ b/config/create-admin.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +USERN=admin +PASSWD='plain$$admin' +TOKEN=$(head -c16 1: - passwd = sys.argv[1] -else: - passwd = getpass.getpass() - -print(generate_password_hash(passwd)) diff --git a/config/gunicorn-frontend-config.py b/config/gunicorn-frontend-config.py index d157a87..1611a15 100644 --- a/config/gunicorn-frontend-config.py +++ b/config/gunicorn-frontend-config.py @@ -1,17 +1,17 @@ -# on which port to listen on (define 'keyfile' and 'certfile' for ssl): -bind = ['0.0.0.0:443'] +# by default, we listen on port 80, without HTTPS! +bind = ['0.0.0.0:80'] +# either use this software behind a reverse proxy, or +# configure TLS and use subscriptions-port80.service: +#bind = ['0.0.0.0:443'] +#certfile = "/etc/letsencrypt/live/subscriptions.gir.st/fullchain.pem" +#keyfile = "/etc/letsencrypt/live/subscriptions.gir.st/privkey.pem" accesslog = None errorlog = "/tmp/frontend21.err" -workers = 1 - -# Note: if you don't have eventlet and 'gunicorn[eventlet]' installed, use -# gthread (below) instead. -worker_class = "eventlet" -worker_connections = 1000 # for eventlet, gevent -#worker_class = "gthread" -#threads = 40 # for gthread +# Note: We are using the in-memory requests-cache, do not use more than 1 +# process, or the cache will be useless! +workers = 1 -certfile = "/etc/letsencrypt/live/subscriptions.gir.st/fullchain.pem" -keyfile = "/etc/letsencrypt/live/subscriptions.gir.st/privkey.pem" +worker_class = "gthread" +threads = 40 -- 2.39.3