leylines/leylines-ansible/playbook-setup.yml

---
- hosts: all
  environment:
    PATH: '/home/{{ ansible_user }}/.local/bin:{{ ansible_env.PATH }}'
    LD_LIBRARY_PATH: '/home/{{ ansible_user }}/.local/lib:{{ ansible_env.get("LD_LIBRARY_PATH", "") }}'
  vars:
    python_version: 3.9.5
    # z3_version: 4.8.11
    base_packages:
      - dask
      - distributed
      - blosc
      - lz4
    worker_packages:
      - numpy
      - scipy
      - pandas
      - scikit-image
      - matplotlib
      - opt_einsum
      - cloudpickle
      - fsspec
      - partd
      - psutil
      - sqlalchemy
      - toolz
      - xxhash
      - tqdm
      - angr
      - h5py
      - zarr
    scheduler_packages:
      - bokeh
      - jupyter-server-proxy
  tasks:
    - name: install system packages
      become: yes
      apt:
        name: ["build-essential", "wget", "software-properties-common", "libnss3-dev", "zlib1g-dev",
               "libgdbm-dev", "libncurses5-dev", "libssl-dev", "libffi-dev", "libreadline-dev",
               "libsqlite3-dev", "libbz2-dev", "libopenblas-dev", "cmake"]
        state: present
        update_cache: yes

    - name: build and install python {{ python_version }}
      register: pyinstall
      changed_when: "'NO_COMPILE_NEEDED' not in pyinstall.stdout"
      args:
        executable: "/bin/bash"
      shell: |
        set -eo pipefail
        existing_ver="$(python3 -c 'import sys;print(f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}")')"
        if [ "$existing_ver" == "{{ python_version }}" ]; then
          echo "NO_COMPILE_NEEDED"
          exit
        fi
        if [ ! -d $HOME/python ]; then
          mkdir $HOME/python
        fi
        cd $HOME/python
        wget -O python.tgz https://www.python.org/ftp/python/{{ python_version }}/Python-{{ python_version }}.tgz
        tar xf python.tgz
        cd Python-{{ python_version }}
        # https://bugs.python.org/issue41346
        sed -i 's/-j0 //' Makefile.pre.in
        # Speed up LTO
        sed -i -e "s|-flto |-flto=4 |g" configure configure.ac
        export CFLAGS="-O3 -fno-semantic-interposition"
        export LDFLAGS="-fno-semantic-interposition"
        ./configure --prefix=$HOME/.local/ \
                    --enable-shared \
                    --with-computed-gotos \
                    --enable-optimizations \
                    --with-lto \
                    --enable-ipv6 \
                    --with-dbmliborder=gdbm:ndbm \
                    --enable-loadable-sqlite-extensions \
                    --with-tzpath=/usr/share/zoneinfo
        make -j$(nproc) EXTRA_CFLAGS="$CFLAGS"
        make EXTRA_CFLAGS="$CFLAGS" install

    # - name: build and install z3 (workers)
    #   when: leylines_is_server == "no"
    #   register: z3install
    #   changed_when: "'NO_COMPILE_NEEDED' not in pyinstall.stdout"
    #   args:
    #     executable: "/bin/bash"
    #   shell: |
    #     set -eo pipefail
    #     existing_ver="$(python3 -c "import z3; print(z3.get_version_string())" || true)"
    #     if [ "$existing_ver" == "{{ z3_version }}" ]; then
    #       echo "NO_COMPILE_NEEDED"
    #       exit
    #     fi
    #     cd $HOME/python
    #     wget -O z3.tgz https://github.com/Z3Prover/z3/archive/z3-{{ z3_version }}.tar.gz
    #     tar xf z3.tgz
    #     # microsoft why are you like this
    #     cd z3-z3-{{ z3_version }}
    #     mkdir -p build
    #     cd build
    #     cmake .. \
    #       -DCMAKE_BUILD_TYPE=Release \
    #       -DCMAKE_INSTALL_PREFIX=$HOME/.local \
    #       -DCMAKE_INSTALL_LIBDIR=lib \
    #       -DZ3_LINK_TIME_OPTIMIZATION=1 \
    #       -DZ3_BUILD_PYTHON_BINDINGS=1 \
    #       -DPYTHON_EXECUTABLE=$HOME/.local/bin/python3
    #     make -j$(nproc)
    #     make install
    #     cd $HOME/.local/lib/python*/site-packages/z3
    #     python3 -m compileall .
    #     python3 -O -m compileall .

    - name: create dask dir
      file:
        path: /home/{{ ansible_user }}/dask
        mode: 0755
        state: directory

    - name: install python packages
      pip:
        virtualenv: /home/{{ ansible_user }}/dask/dask-venv
        virtualenv_command: python3 -m venv
        state: latest
        name: "{% if leylines_is_server == 'yes' %}{{ base_packages + scheduler_packages }}{% else %}{{ base_packages + worker_packages }}{% endif %}"

    # - name: create alternate venv for angr
    #   when: leylines_is_server == "no"
    #   pip:
    #     virtualenv: /home/{{ ansible_user }}/dask/angr-venv
    #     virtualenv_command: python3 -m venv
    #     state: latest
    #     name: ["angr", "cloudpickle"]
    #
    # - name: copy weird packages
    #   when: leylines_is_server == "no"
    #   changed_when: true
    #   args:
    #     executable: "/bin/bash"
    #   shell: |
    #     set -eo pipefail
    #     cp -r $HOME/.local/lib/python*/site-packages/z3/ $HOME/dask/dask-venv/lib/python*/site-packages/

    - name: install systemd user dir
      file:
        path: /home/{{ ansible_user }}/.config/systemd/user
        mode: 0755
        state: directory

    - name: install systemd task
      template:
        mode: 0644
        src: templates/leylines-{% if leylines_is_server == "yes" %}scheduler{% else %}worker{% endif %}.service
        dest: /home/{{ ansible_user }}/.config/systemd/user

    - name: enable linger
      command: loginctl enable-linger
      changed_when: true

    - name: enable and start systemd task
      systemd:
        scope: user
        daemon_reload: true
        name: leylines-{% if leylines_is_server == "yes" %}scheduler{% else %}worker{% endif %}
        enabled: yes
        state: restarted