From eda9fa47b0db2f952efa1756550da7c699e6d64c Mon Sep 17 00:00:00 2001 From: haskal Date: Fri, 18 Jun 2021 03:53:27 -0400 Subject: [PATCH] add better instructions and nginx support files --- README.md | 95 +++++++++++++++++++----- leylines-support/leylines-daemon.service | 2 + leylines-support/nginx-http.conf | 16 ++++ leylines-support/nginx.conf | 13 ++++ leylines-support/options-ssl-stream.conf | 9 +++ 5 files changed, 117 insertions(+), 18 deletions(-) create mode 100644 leylines-support/nginx-http.conf create mode 100644 leylines-support/nginx.conf create mode 100644 leylines-support/options-ssl-stream.conf diff --git a/README.md b/README.md index caff4b1..60ac970 100644 --- a/README.md +++ b/README.md @@ -8,25 +8,57 @@ deployment for dask nodes ## how to +### install the server ```bash (cd leylines-monocypher && pip3 install --user .) (cd leylines && pip3 install --user .) mkdir -p ~/.config/leylines +``` + +ok now take a moment to edit `leylines-support/leylines-daemon.service` to be running as your user +(change `User=` and `Group=`). put that into your `/etc/systemd/system` and then do + +```bash +sudo systemctl --enable now leylines-daemon +``` + +congrats wireguard should be up. next, edit `leylines-support/nginx.conf` (change the listen address +and the SSL certificate paths -- point those towards letsencrypt directories for a domain you +already provisioned that your nginx is serving). put that block into your `/etc/nginx/nginx.conf`. +to export your dask dashboard publicly, also adjust `leylines-support/nginx-http.conf` to your needs +and include it in an http server block. it may be advantageous to do that first, then run `certbot` +on the domain to get the certs provisioned, and then set up the `stream` block using the same certs +as certbot inserted for https + +then run +```bash +sudo nginx -s reload +``` + +### install client + +now that the server is running, you may choose to access it remotely. make a note of `leylines +print-token` -- this is the auth token you will need. on your client (local laptop, or something) + +```bash +(cd leylines && pip3 install --user .) +mkdir -p ~/.config/leylines +echo "auth token here" > ~/.config/leylines/token +echo "mycluster.domain.lgbt" > ~/.config/leylines/host +``` + +now you can access your server using the CLI. initialize it and add some nodes. in the `init` +command provide the server's externally-facing public IP, and provide an SSH key that can be used to +access it for ansible. then, to add workers provide a name for each one and an SSH key + +```bash leylines init -n myserver -i 1.2.3.4 -k path/to/ssh-key leylines add -n worker-0 -k path/to/ssh-key ... leylines add -n worker-n -k path/to/ssh-key ``` -optionally copy the database to your laptop so you can run ansible locally (there will be some -actual API soon but not right now) - -start a privileged shell (there is no service for the wireguard stuff yet -- coming soon) -```bash -systemd-run -tS --uid $(id -u) --gid $(id -g) -pAmbientCapabilities=CAP_NET_ADMIN -``` - -sync wireguard settings +sync wireguard settings (this applies the configuration to the server's wireguard interface) ```bash leylines sync ``` @@ -36,6 +68,8 @@ get status leylines status ``` +### connect a worker + get config for a node ```bash leylines get-conf @@ -46,20 +80,23 @@ manually copy that config to your worker node, `/etc/wireguard/leyline-wg.conf` currently the wireguard topology is a star. this doesn't actually work optimally for my config, where some nodes are colocated and should have direct connections to each other and others should go -over WAN to reach distant nodes. this will be changed (you may be sensing a pattern with the amount -of TODO) +over WAN to reach distant nodes. this will be changed in a later version + +### provision workers + +run the ansible playbook. this will provision the needed components for dask on the server and all +workers -run the ansible playbook ```bash cd leylines-ansible ansible-playbook -i leylines_inv.py playbook-setup.yml ``` the first run will take a while. it builds python 3.9.5 and installs it, then builds a virtualenv -with python dependencies in it, and then installs and starts systemd user services for the scheduler -and workers +with python dependencies in it, and then installs and starts systemd user services -now you can open `:31336` to view the dask dashboard +now you can open `:31336` to view the dask dashboard (or if you are +proxying it with nginx, it should be available there too) use the cluster with ```python @@ -67,6 +104,28 @@ from dask.distributed import Client client = Client(":31337") ``` +or more easily +```python +from leylines.dask import init_dask +client = init_dask() +``` + +or +```python +from leylines.dask import init_dask_async +client = await init_dask_async +``` + +`leylines.dask` also provides `tqdmprogress` which can be used in the place of +`distributed.diagnostics.progress` for a task monitor using `tqdm`, and `tqdm_await` which can be +used with an iterable of dask futures to display progress as they go (but only for async clients) + +```python +futures = [ some list of futures ... ] +async for fut in tqdm_await(futures, pbar=): + print(fut.result()) +``` + ### time for magic copy `leylines-support/02-dask.py` into `~/.ipython/profile_default/startup` @@ -75,6 +134,6 @@ this provides 2 new spells: `%dask` connects to your cluster, and `%daskworker` ipython console on a worker selected by having free RAM available and not being busy. this is useful for ad-hoc code testing on a real worker -%dask also installs `client`, a reference to the client, and `tqdmprogress`, which can be used in -place of `distributed.diagnostics.progress` for a task monitor using `tqdm`, and `upload` which -uploads a file and returns a delayed function which will fetch the filename on a worker +%dask also installs `client`, a reference to the client, and `tqdmprogress` from `leylines.dask`, +and `upload` which uploads a file and returns a delayed function which will fetch the filename on a +worker diff --git a/leylines-support/leylines-daemon.service b/leylines-support/leylines-daemon.service index 0248b26..c955af9 100644 --- a/leylines-support/leylines-daemon.service +++ b/leylines-support/leylines-daemon.service @@ -5,6 +5,8 @@ Description=leylines server Type=simple ExecStart=/usr/bin/env python3 -m leylines daemon AmbientCapabilities=CAP_NET_ADMIN +User=youruser +Group=youruser [Install] WantedBy=default.target diff --git a/leylines-support/nginx-http.conf b/leylines-support/nginx-http.conf new file mode 100644 index 0000000..fd7c610 --- /dev/null +++ b/leylines-support/nginx-http.conf @@ -0,0 +1,16 @@ +server { + server_name your domain name; + + location / { + proxy_pass http://:31336; + proxy_set_header Authorization ""; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + proxy_set_header Host $host; + auth_basic "awoo systems -- authorization required"; + auth_basic_user_file put an htpasswd file here; + } + + listen 80; +} diff --git a/leylines-support/nginx.conf b/leylines-support/nginx.conf new file mode 100644 index 0000000..36a3adf --- /dev/null +++ b/leylines-support/nginx.conf @@ -0,0 +1,13 @@ +stream { + upstream leylines { + server 127.0.0.1:31338; + } + server { + listen your_public_ip:31337 ssl; + proxy_pass leylines; + ssl_certificate /etc/letsencrypt/live/your_domain_name/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/your_domain_name/privkey.pem; + include options-ssl-stream.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; + } +} diff --git a/leylines-support/options-ssl-stream.conf b/leylines-support/options-ssl-stream.conf new file mode 100644 index 0000000..c77f237 --- /dev/null +++ b/leylines-support/options-ssl-stream.conf @@ -0,0 +1,9 @@ +# Adapted from let's encrypt settings for HTTP + +ssl_session_cache shared:le_nginx_stream_SSL:1m; +ssl_session_timeout 1440m; + +ssl_protocols TLSv1 TLSv1.1 TLSv1.2; +ssl_prefer_server_ciphers on; + +ssl_ciphers "ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS";