Add mastodon

README.md

@@ -16,6 +16,8 @@ Bloodletting:
     - `fail2ban`
     - `grafana`
     - `homepage`
+    - `mastodon`
+      - dump and import Postgres and Redis DBs
     - `matterbridge`
     - `mc-e2e`
     - `mstdn-ebooks`

common/fragments/bin.nix

@@ -5,6 +5,6 @@
     enable = true;
     address = "0.0.0.0";
     port = 6162;
-    textUploadLimit = 32;
+    textUploadLimit = 64;
   };
 }

common/fragments/mastodon.nix

@@ -0,0 +1,24 @@
+{ config, pkgs, ... }: {
+  services.mastodon = {
+    enable = true;
+
+    package = pkgs.agatha-mastodon;
+
+    localDomain = "technogothic.net";
+    configureNginx = false;
+    smtp.fromAddress = "noreply@technogothic.net";
+    smtp.createLocally = false;
+    database.passwordFile = "/var/lib/mastodon/secrets/db-password";
+    extraConfig = {
+      WEB_DOMAIN = "fv.technogothic.net";
+      GITHUB_REPOSITORY = "AgathaSorceress/mastodon";
+      AUTHORIZED_FETCH = "true";
+      MAX_TOOT_CHARS = "6666";
+      MAX_POLL_OPTIONS = "128";
+      MAX_POLL_OPTION_CHARS = "512";
+      EXTRA_DATA_HOSTS = "https://ftp.technogothic.net";
+    };
+  };
+
+  users.groups.mastodon.members = [ config.services.nginx.user ];
+}

common/fragments/postgres.nix

@@ -0,0 +1,22 @@
+{ config, ... }: {
+  services.postgresql = {
+    settings = {
+      max_connections = 200;
+      shared_buffers = "4GB";
+      effective_cache_size = "12GB";
+      maintenance_work_mem = "1GB";
+      checkpoint_completion_target = 0.9;
+      wal_buffers = "16MB";
+      default_statistics_target = 100;
+      random_page_cost = 1.1;
+      effective_io_concurrency = 300;
+      work_mem = "10485kB";
+      min_wal_size = "1GB";
+      max_wal_size = "4GB";
+      max_worker_processes = 4;
+      max_parallel_workers_per_gather = 2;
+      max_parallel_workers = 4;
+      max_parallel_maintenance_workers = 2;
+    };
+  };
+}

common/pkgs/mastodon/default.nix

@@ -0,0 +1,159 @@
+{ lib, stdenv, nodejs-slim, mkYarnPackage, fetchFromGitHub, bundlerEnv
+, nixosTests, yarn, callPackage, imagemagick, ffmpeg, file, ruby_3_0
+, writeShellScript, fetchYarnDeps, fixup_yarn_lock, brotli
+
+# Allow building a fork or custom version of Mastodon:
+, pname ? "mastodon", version ? import ./version.nix, srcOverride ? null
+, dependenciesDir ? ./. # Should contain gemset.nix, yarn.nix and package.json.
+}:
+
+stdenv.mkDerivation rec {
+  inherit pname version;
+
+  # Using overrideAttrs on src does not build the gems and modules with the overridden src.
+  # Putting the callPackage up in the arguments list also does not work.
+  src =
+    if srcOverride != null then srcOverride else callPackage ./source.nix { };
+
+  mastodonGems = bundlerEnv {
+    name = "${pname}-gems-${version}";
+    inherit version;
+    ruby = ruby_3_0;
+    gemdir = src;
+    gemset = dependenciesDir + "/gemset.nix";
+    # This fix (copied from https://github.com/NixOS/nixpkgs/pull/76765) replaces the gem
+    # symlinks with directories, resolving this error when running rake:
+    #   /nix/store/451rhxkggw53h7253izpbq55nrhs7iv0-mastodon-gems-3.0.1/lib/ruby/gems/2.6.0/gems/bundler-1.17.3/lib/bundler/settings.rb:6:in `<module:Bundler>': uninitialized constant Bundler::Settings (NameError)
+    postBuild = ''
+      for gem in "$out"/lib/ruby/gems/*/gems/*; do
+        cp -a "$gem/" "$gem.new"
+        rm "$gem"
+        # needed on macOS, otherwise the mv yields permission denied
+        chmod +w "$gem.new"
+        mv "$gem.new" "$gem"
+      done
+    '';
+  };
+
+  mastodonModules = stdenv.mkDerivation {
+    pname = "${pname}-modules";
+    inherit src version;
+
+    yarnOfflineCache = fetchYarnDeps {
+      yarnLock = "${src}/yarn.lock";
+      sha256 = "sha256-Y6ORZE18/qeVEvuzbx+tPe8Pjjxvf7+mHwspW3pmuEg=";
+    };
+
+    nativeBuildInputs = [
+      fixup_yarn_lock
+      nodejs-slim
+      yarn
+      mastodonGems
+      mastodonGems.wrappedRuby
+      brotli
+    ];
+
+    RAILS_ENV = "production";
+    NODE_ENV = "production";
+
+    buildPhase = ''
+      export HOME=$PWD
+      # This option is needed for openssl-3 compatibility
+      # Otherwise we encounter this upstream issue: https://github.com/mastodon/mastodon/issues/17924
+      export NODE_OPTIONS=--openssl-legacy-provider
+      fixup_yarn_lock ~/yarn.lock
+      yarn config --offline set yarn-offline-mirror $yarnOfflineCache
+      yarn install --offline --frozen-lockfile --ignore-engines --ignore-scripts --no-progress
+
+      patchShebangs ~/bin
+      patchShebangs ~/node_modules
+
+      # skip running yarn install
+      rm -rf ~/bin/yarn
+
+      OTP_SECRET=precompile_placeholder SECRET_KEY_BASE=precompile_placeholder \
+        rails assets:precompile
+      yarn cache clean --offline
+      rm -rf ~/node_modules/.cache
+
+      # Create missing static gzip and brotli files
+      gzip --best --keep ~/public/assets/500.html
+      gzip --best --keep ~/public/packs/report.html
+      find ~/public/assets -maxdepth 1 -type f -name '.*.json' \
+        -exec gzip --best --keep --force {} ';'
+      brotli --best --keep ~/public/packs/report.html
+      find ~/public/assets -type f -regextype posix-extended -iregex '.*\.(css|js|json|html)' \
+        -exec brotli --best --keep {} ';'
+    '';
+
+    installPhase = ''
+      mkdir -p $out/public
+      cp -r node_modules $out/node_modules
+      cp -r public/assets $out/public
+      cp -r public/packs $out/public
+    '';
+  };
+
+  propagatedBuildInputs = [ imagemagick ffmpeg file mastodonGems.wrappedRuby ];
+  buildInputs = [ mastodonGems nodejs-slim ];
+
+  buildPhase = ''
+    ln -s $mastodonModules/node_modules node_modules
+    ln -s $mastodonModules/public/assets public/assets
+    ln -s $mastodonModules/public/packs public/packs
+
+    patchShebangs bin/
+    for b in $(ls $mastodonGems/bin/)
+    do
+      if [ ! -f bin/$b ]; then
+        ln -s $mastodonGems/bin/$b bin/$b
+      fi
+    done
+
+    # Remove execute permissions
+    chmod 0444 public/emoji/*.svg
+
+    # Create missing static gzip and brotli files
+    find public -maxdepth 1 -type f -regextype posix-extended -iregex '.*\.(css|js|svg|txt|xml)' \
+      -exec gzip --best --keep --force {} ';' \
+      -exec brotli --best --keep {} ';'
+    find public/emoji -type f -name '.*.svg' \
+      -exec gzip --best --keep --force {} ';' \
+      -exec brotli --best --keep {} ';'
+    ln -s assets/500.html.gz public/500.html.gz
+    ln -s assets/500.html.br public/500.html.br
+    ln -s packs/sw.js.gz public/sw.js.gz
+    ln -s packs/sw.js.br public/sw.js.br
+    ln -s packs/sw.js.map.gz public/sw.js.map.gz
+    ln -s packs/sw.js.map.br public/sw.js.map.br
+
+    rm -rf log
+    ln -s /var/log/mastodon log
+    ln -s /tmp tmp
+  '';
+
+  installPhase = let
+    run-streaming = writeShellScript "run-streaming.sh" ''
+      # NixOS helper script to consistently use the same NodeJS version the package was built with.
+      ${nodejs-slim}/bin/node ./streaming
+    '';
+  in ''
+    mkdir -p $out
+    cp -r * $out/
+    ln -s ${run-streaming} $out/run-streaming.sh
+  '';
+
+  passthru = {
+    tests.mastodon = nixosTests.mastodon;
+    updateScript = callPackage ./update.nix { };
+  };
+
+  meta = with lib; {
+    description =
+      "Self-hosted, globally interconnected microblogging software based on ActivityPub";
+    homepage = "https://joinmastodon.org";
+    license = licenses.agpl3Plus;
+    platforms = [ "x86_64-linux" "i686-linux" "aarch64-linux" ];
+    maintainers = with maintainers; [ happy-river erictapen izorkin ghuntley ];
+  };
+}

common/pkgs/mastodon/source.nix

@@ -0,0 +1,11 @@
+# This file was generated by pkgs.mastodon.updateScript.
+{ fetchgit, applyPatches }: let
+  src = fetchgit {
+    url = "https://github.com/AgathaSorceress/mastodon.git";
+    rev = "dba9fc96761d19f65b519792cd469c3a1859d1db";
+    sha256 = "0qismh128j16b0vfq9jfc37i68xnmbza066r5r4ynrgfyrcfzsdk";
+  };
+in applyPatches {
+  inherit src;
+  patches = [];
+}

common/pkgs/mastodon/update.nix

@@ -0,0 +1,27 @@
+{ pkgs }:
+let
+  binPath = with pkgs;
+    lib.makeBinPath [
+      yarn2nix
+      bundix
+      coreutils
+      diffutils
+      nix-prefetch-git
+      gnused
+      jq
+    ];
+in pkgs.runCommand "mastodon-update-script" {
+  nativeBuildInputs = [ pkgs.makeWrapper ];
+
+  meta = {
+    maintainers = with pkgs.lib.maintainers; [ happy-river ];
+    description =
+      "Utility to generate Nix expressions for Mastodon's dependencies";
+    platforms = pkgs.lib.platforms.unix;
+  };
+} ''
+  mkdir -p $out/bin
+  cp ${./update.sh} $out/bin/update.sh
+  patchShebangs $out/bin/update.sh
+  wrapProgram $out/bin/update.sh --prefix PATH : ${binPath}
+''

common/pkgs/mastodon/update.sh

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+set -e
+
+URL=https://github.com/AgathaSorceress/mastodon.git
+
+POSITIONAL=()
+while [[ $# -gt 0 ]]; do
+    key="$1"
+
+    case $key in
+        --url)
+            URL="$2"
+            shift # past argument
+            shift # past value
+            ;;
+        --ver)
+            VERSION="$2"
+            shift # past argument
+            shift # past value
+            ;;
+        --rev)
+            REVISION="$2"
+            shift # past argument
+            shift # past value
+            ;;
+        --patches)
+            PATCHES="$2"
+            shift # past argument
+            shift # past value
+            ;;
+        *)  # unknown option
+            POSITIONAL+=("$1")
+            shift # past argument
+            ;;
+    esac
+done
+
+if [[ -z "$VERSION" || -n "$POSITIONAL" ]]; then
+    echo "Usage: update.sh [--url URL] --ver VERSION [--rev REVISION] [--patches PATCHES]"
+    echo "URL may be any path acceptable to 'git clone' and VERSION the"
+    echo "semantic version number.  If VERSION is not a revision acceptable to"
+    echo "'git checkout', you must provide one in REVISION.  If URL is not"
+    echo "provided, it defaults to https://github.com/mastodon/mastodon.git."
+    echo "PATCHES, if provided, should be one or more Nix expressions"
+    echo "separated by spaces."
+    exit 1
+fi
+
+if [[ -z "$REVISION" ]]; then
+    REVISION="$VERSION"
+fi
+
+rm -f gemset.nix version.nix source.nix
+TARGET_DIR="$PWD"
+
+
+WORK_DIR=$(mktemp -d)
+
+# Check that working directory was created.
+if [[ ! "$WORK_DIR" || ! -d "$WORK_DIR" ]]; then
+    echo "Could not create temporary directory"
+    exit 1
+fi
+
+# Delete the working directory on exit.
+function cleanup {
+    # Report errors, if any, from nix-prefetch-git
+    grep "fatal" $WORK_DIR/nix-prefetch-git.out >/dev/stderr || true
+    rm -rf "$WORK_DIR"
+}
+trap cleanup EXIT
+
+echo "Fetching source code $REVISION from $URL"
+JSON=$(nix-prefetch-git --url "$URL" --rev "$REVISION"  2> $WORK_DIR/nix-prefetch-git.out)
+SHA=$(echo $JSON | jq -r .sha256)
+FETCHED_SOURCE_DIR=$(grep '^path is' $WORK_DIR/nix-prefetch-git.out | sed 's/^path is //')
+
+echo "Creating version.nix"
+echo \"$VERSION\" | sed 's/^"v/"/' > version.nix
+
+cat > source.nix << EOF
+# This file was generated by pkgs.mastodon.updateScript.
+{ fetchgit, applyPatches }: let
+  src = fetchgit {
+    url = "$URL";
+    rev = "$REVISION";
+    sha256 = "$SHA";
+  };
+in applyPatches {
+  inherit src;
+  patches = [$PATCHES];
+}
+EOF
+SOURCE_DIR="$(nix-build --no-out-link -E '(import <nixpkgs> {}).callPackage ./source.nix {}')"
+
+echo "Creating gemset.nix"
+bundix --lockfile="$SOURCE_DIR/Gemfile.lock" --gemfile="$SOURCE_DIR/Gemfile"
+echo "" >> $TARGET_DIR/gemset.nix  # Create trailing newline to please EditorConfig checks

common/pkgs/mastodon/version.nix

@@ -0,0 +1 @@
+"dba9fc96761d19f65b519792cd469c3a1859d1db"

flake.lock

@@ -527,6 +527,21 @@
         "type": "github"
       }
     },
+    "nixpkgs-unstable": {
+      "locked": {
+        "lastModified": 1682566018,
+        "narHash": "sha256-HPzPRFiy2o/7k7mtnwfM1E6NVZHiFbPdmYCMoIpkHO4=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "8e3b64db39f2aaa14b35ee5376bd6a2e707cadc2",
+        "type": "github"
+      },
+      "original": {
+        "id": "nixpkgs",
+        "ref": "nixpkgs-unstable",
+        "type": "indirect"
+      }
+    },
     "nixpkgsV1": {
       "locked": {
         "lastModified": 1678500271,
@@ -744,6 +759,7 @@
         "home-manager": "home-manager",
         "mms": "mms",
         "nixpkgs": "nixpkgs_8",
+        "nixpkgs-unstable": "nixpkgs-unstable",
         "url-eater": "url-eater",
         "vampysite": "vampysite"
       }

flake.nix

@@ -1,6 +1,7 @@
 {
   inputs = {
     nixpkgs.url = "nixpkgs/nixos-22.11";
+    nixpkgs-unstable.url = "nixpkgs/nixpkgs-unstable";
 
     home-manager = {
       url = "github:nix-community/home-manager";
@@ -24,8 +25,8 @@
 
     helix.url = "github:helix-editor/helix";
   };
-  outputs = { nixpkgs, home-manager, mms, helix, url-eater, colorpickle
+  outputs = { nixpkgs, nixpkgs-unstable, home-manager, mms, helix, url-eater
-    , frq-friend, colmena, vampysite, ... }: {
+    , colorpickle, frq-friend, colmena, vampysite, ... }: {
       colmena = {
         network = {
           description = "Agatha's Nix Infra";
@@ -50,8 +51,11 @@
             ./hosts/bloodletting/configuration.nix
             (import "${home-manager}/nixos")
             mms.module
+            "${nixpkgs-unstable}/nixos/modules/services/web-apps/mastodon.nix"
           ];
 
+          disabledModules = [ "services/web-apps/mastodon.nix" ];
+
           deployment = {
             targetUser = "root";
             targetHost = "bloodletting";

hosts/bloodletting/configuration.nix

@@ -8,16 +8,22 @@
     ../../common/fragments/grafana.nix
     ../../common/fragments/homepage.nix
     ../../common/fragments/mastodon-ebooks.nix
+    ../../common/fragments/mastodon.nix
     ../../common/fragments/matterbridge.nix
     ../../common/fragments/minecraft.nix
     ../../common/fragments/nyandroid.nix
+    ../../common/fragments/postgres.nix
     ../../common/fragments/prometheus_exporters.nix
     ../../common/fragments/vsftpd.nix
     ../../common/home_manager/common.nix
   ];
 
   nixpkgs.overlays = [
-    (final: prev: { bin = final.callPackage ../../common/pkgs/bin.nix { }; })
+    (final: prev: {
+      bin = final.callPackage ../../common/pkgs/bin.nix { };
+      agatha-mastodon =
+        final.callPackage ../../common/pkgs/mastodon/default.nix { };
+    })
   ];
 
   # Bootloader.
@@ -103,6 +109,17 @@
 
       locations."=/cv.pdf" = { alias = "/home/ftp/cv.pdf"; };
 
+      locations."=/.well-known/host-meta" = {
+        return = "301 https://fv.technogothic.net$request_uri";
+      };
+
+      locations."=/.well-known/webfinger" = {
+        return = "301 https://fv.technogothic.net$request_uri";
+        extraConfig = ''
+          add_header Access-Control-Allow-Origin '*';
+        '';
+      };
+
       extraConfig = ''
         error_page 404 /404.html;
       '';
@@ -141,6 +158,9 @@
       locations."/" = {
         proxyPass = "http://localhost:6162";
         proxyWebsockets = true;
+        extraConfig = "client_max_body_size ${
+            toString config.services.bin.textUploadLimit
+          }M;";
       };
     };
 
@@ -152,6 +172,27 @@
 
       locations."/" = { extraConfig = "autoindex on;"; };
     };
+
+    virtualHosts."fv.technogothic.net" = {
+      useACMEHost = "technogothic.net";
+      forceSSL = true;
+
+      root = "${config.services.mastodon.package}/public/";
+
+      locations."/system/".alias = "/var/lib/mastodon/public-system/";
+
+      locations."/" = { tryFiles = "$uri @proxy"; };
+
+      locations."@proxy" = {
+        proxyPass = "http://unix:/run/mastodon-web/web.socket";
+        proxyWebsockets = true;
+      };
+
+      locations."/api/v1/streaming/" = {
+        proxyPass = "http://unix:/run/mastodon-streaming/streaming.socket";
+        proxyWebsockets = true;
+      };
+    };
   };
 
   # This value determines the NixOS release from which the default