From 10768ce069f5c08e8e4393c494f0c6900922170c Mon Sep 17 00:00:00 2001 From: Agatha Lovelace Date: Thu, 16 Feb 2023 13:50:41 +0100 Subject: [PATCH] Add post: Running Jekyll on Nix --- source/_pages/JekyllOnNix.md | 431 +++++++++++++++++++++++++++++++++++ 1 file changed, 431 insertions(+) create mode 100644 source/_pages/JekyllOnNix.md diff --git a/source/_pages/JekyllOnNix.md b/source/_pages/JekyllOnNix.md new file mode 100644 index 0000000..cf05877 --- /dev/null +++ b/source/_pages/JekyllOnNix.md @@ -0,0 +1,431 @@ +--- +title: Running a Jekyll website on Nix +--- + +If you're reading this on [technogothic.net](/) and the year is something like 2023, this site is probably generated by Jekyll. +Previously this website was hosted in an Arch Linux VM, then in a Debian 11 LXC container on top of Proxmox, but on both systems it was deployed and built manually with scp and bundler. +The previous two systems became more and more unmaintainable over the years, so a few months ago I started migrating my servers to NixOS. +This website was the last thing I migrated, and one of the most painful parts, by far. + +# Previous attempts +So, as with all things, I started by typing `build jekyll website on nix` into my browser's search bar and pressing Enter. +The first results that came up were: + - Building a Jekyll Environment with NixOS [*(stesie.github.io, 2016)*](https://stesie.github.io/2016/08/nixos-github-pages-env) + - Jekyll setup in NixOS [*(savo.rocks, 2017)*](https://savo.rocks/posts/jekyll-setup-in-nixos/) + - Using Jekyll and Nix to blog [*(nathan.gs, 2019)*](https://nathan.gs/2019/04/19/using-jekyll-and-nix-to-blog/) + - Getting Jekyll to Work in NixOS [*(matthewrhone.dev, 2021)*](https://matthewrhone.dev/jekyll-in-nixos) + - Jekyll Development Environment on Nix [*(suffix.be, 2022)*](https://www.suffix.be/blog/bundling-jekyll-nixos/) + +To quickly summarize these articles, they have these steps in common: + +First you have to run these commands: +```bash +bundler lock +bundler package --path vendor/cache --no-install +nix-shell -p bundix +bundix +``` +This creates a bundler lockfile, fetches the dependencies into `vendor/cache` (without installing) and uses `bundix` to generate `gemset.nix`, a file used by Nix's `bundlerEnv` to know which gems we need. +Then, we create `default.nix`, where we define the environment that `nix-shell` will later drop us into: + +```nix +with (import {}); +let + env = bundlerEnv { + name = "your-package"; + inherit ruby; + gemfile = ./Gemfile; + lockfile = ./Gemfile.lock; + gemset = ./gemset.nix; + }; +in stdenv.mkDerivation { + name = "your-package"; + buildInputs = [env ruby]; +} +``` + +Clearly, these steps worked at some point between 2016 and 2022, as the blog post authors probably wouldn't have published these posts if it didn't. +So, sounds simple enough, right? +
+
+
+*Wrong* +
+ +# My first attempt + +```nix +{ pkgs, lib, ... }: +let + version = "a6476f6fc94f4476f31ebd35687dd54a807d41dd"; + repo = pkgs.fetchgit { + url = "https://git.lain.faith/sorceress/vampysite.git"; + rev = version; + sha256 = "1lr5jqsqik2czgas7ci4fynavhkcw6nfiw1p3lwlxm0ar3y483fk"; + }; + + jekyll_env = patched_pkgs.bundlerEnv { + name = "jekyll_env"; + ruby = pkgs.ruby; + + gemdir = "${repo}/."; + }; +in pkgs.stdenv.mkDerivation { + inherit version; + name = "vampysite"; + + src = repo; + + buildInputs = with pkgs; [ + jekyll_env + jekyll_env.wrappedRuby + ]; + + buildPhase = '' + bundle exec jekyll build + ''; + + installPhase = '' + mkdir -p $out + cp -r _site/* $out/ + ''; +} +``` + +Time-traveling hashes aside, this was the first version that more or less looked like it could maybe work. +The main differences here are: +- I would like to package the website as a derivation to serve from my NixOS host, not have a development environment for Jekyll (though, after getting this to work, making one was quite easy as well). +- The sources for the website are fetched from git, rather than being in the same directory. +- It's now possible to specify just the `gemdir` in `bundlerEnv` without manually writing out each path. + +The first issue I ran into was building the native extensions for the Nokogiri gem. It was missing a dependency, so let's add that to `gemset.nix`: +```nix + mini_portile2 = { + groups = [ "default" ]; + platforms = [ ]; + source = { + remotes = [ "https://rubygems.org" ]; + sha256 = + "b70e325e37a378aea68b6d78c9cdd060c66cbd2bef558d8f13a6af05b3f2c4a9"; + type = "gem"; + }; + version = "2.8.1"; + }; +``` +And then change the `nokogiri` definition above: +```diff + nokogiri = { +- dependencies = [ "racc" ]; ++ dependencies = [ "racc" "mini_portile2" ]; + groups = [ "default" ]; + platforms = [ ]; + source = { + remotes = [ "https://rubygems.org" ]; + sha256 = "sts693acKc131fOc09C2WrEJdb3s8EvnHWg/nJq+JmM="; + type = "gem"; + }; + version = "1.14.1"; + }; +``` + +I also added Nokogiri's dependencies to my build inputs: +```nix + buildInputs = with pkgs; [ + jekyll_env + # nokogiri dependencies + zlib + libiconv + libxml2 + libxslt + ]; +``` + +Other than that, I had to retry building a few times and fix up some incorrect sha256 hashes outputted by bundix. +Now that Nokogiri builds, the next issue I ran into was building `sass-embedded`: + +``` +building '/nix/store/04rabbrj3wgk7nxahz87wq0nsnyzl9c7-ruby2.7.6-sass-embedded-1.58.0.drv'... + ... +Building native extensions. This could take a while... +ERROR: Error installing /nix/store/x4dvi7mffypdbm3755vy221hjc2bapd9-sass-embedded-1.58.0.gem: + ERROR: Failed to build gem native extension. + + current directory: /nix/store/rf9dnjclddg2ksyhycakq6jizkshv7dx-ruby2.7.6-sass-embedded-1.58.0/lib/ruby/gems/2.7.0/gems/sass-embedded-1.58.0/ext/sass + ... +fetch https://github.com/sass/dart-sass-embedded/releases/download/1.58.0/sass_embedded-1.58.0-linux-x64.tar.gz +rake aborted! +SocketError: Failed to open TCP connection to github.com:443 (getaddrinfo: Temporary failure in name resolution) + ... + +Caused by: +SocketError: getaddrinfo: Temporary failure in name resolution + ... +``` + +The build fails due to the gem Rakefile trying to download a tarball from GitHub during build time, which is not allowed in Nix derivations (and quite a lot of build systems, actually). +As I found out by opening [this](https://github.com/ntkme/sass-embedded-host-ruby/issues/102) issue, it's possible to override the path for the tarball and avoid the download. +As this was ~6 hours into the process, I decided to not bother with it for now and instead went with a simpler workaround: + +```diff +- gem "jekyll", "~> 4.3" ++ gem "jekyll", "~> 4.2.2" + gem "jekyll-toc", "~> 0.17.1" + gem "kramdown-math-katex", "~> 1.0" +- gem "rouge", "~> 4.0" ++ gem "rouge", "~> 3.0" + gem "webrick", "~> 1.7" + gem "image_optim", "~> 0.31.2" + gem "image_optim_pack", "~> 0.9.1" +``` + +Downgrading Jekyll replaced `sass-embedded` with `sassc`, which builds successfully with no further tweaks. + +Then, the environment built successfully, but upon running `bundle exec jekyll build` I got +``` + Could not find public_suffix-5.0.1 in any of the sources +``` +Which was *interesting* to debug, to say the least. I verified that `bundle env` and `gem environment` have all the paths set correctly and `public_suffix-5.0.1` is installed. +After a few more hours, I added the `jekyll_env` derivation to `$PATH`, which didn't solve the issue, but made the error different. +``` + definition.rb:507:in `materialize': Could not find image_optim_pack-0.9.1.20230129-x86_64-linux, nokogiri-1.14.1-x86_64-linux in locally installed gems +``` +Which is even more interesting, because now it can find `public_suffix`, but not `nokogiri` or `image_optim_pack`, while all 3 gems are installed in the same way. +Next morning, I decided to check `Gemfile.lock` once again, just to be sure that the gems are named correctly. +
+... They are not, not at all. + +The solution was something like this: +```diff +- nokogiri (1.14.1-x86_64_linux) ++ nokogiri (1.14.1) + ... +- image_optim_pack (0.9.1.20230129-x86_64_linux) ++ image_optim_pack (0.9.1.20230129) +``` + +Now, if you're using *only* Jekyll, this is the point where we reach the happy ending. +
+Unfortunately, + +# Unhelpful Error Messages And The Debugging Hell + +As you might have noticed, I have some other dependencies in my `Gemfile`: + - `jekyll-toc` is used to show the Table Of Contents you can see in the top right part of the screen if you scroll up, or at the end of this post, depending on your screen size. + - `image_optim`, and by extension, `image_optim_pack` are used to compress images on the website before serving it. + +`jekyll-toc` builds with no problems, but `image_optim` is, however, completely different: + +```bash +lib/image_optim/bin_resolver/bin.rb:119:in `[]': invalid byte sequence in US-ASCII (ArgumentError) +``` + +I first tried solving this by adding `locale` to build inputs and setting the correct locale settings. This got me to: + +```bash +lib/image_optim/bin_resolver/bin.rb:119:in `[]': invalid byte sequence in UTF-8 (ArgumentError) +``` + +Clearly missing locales is not the problem here. +Digging further, I added `bundle exec image_optim --info` to the build script. The output was a little bit more helpful: + +```nix + ... +image_optim_pack: all bins from /nix/store/36q0biq46yha892ws5hjcnvlyzc74zsx-ruby2.7.6-image_optim_pack-0.9.1.20230129/lib/ruby/gems/2.7.0/gems/image_optim_pack-0.9.1.20230129/vendor/linux-x86_64 failed + ... +``` + +Now, here comes the worse part: Building the same derivation locally on my desktop, with Nix on Arch Linux, worked correctly and the same command gave me this output: + +```nix + ... +image_optim_pack: all bins from /nix/store/36q0biq46yha892ws5hjcnvlyzc74zsx-ruby2.7.6-image_optim_pack-0.9.1.20230129/lib/ruby/gems/2.7.0/gems/image_optim_pack-0.9.1.20230129/vendor/linux-x86_64 worked + ... +``` + +
+Somewhere around here I also tried running `nix-shell --pure` on the system that worked and realized that Jekyll (or one of the plugins?) will want a JS runtime. An easy fix: +```nix + buildInputs = with pkgs; [ + jekyll_env + # nokogiri dependencies + zlib + libiconv + libxml2 + libxslt + # jekyll wants a JS runtime + nodejs-slim + ]; +``` + +
+ +I then made sure that those directories in both environments were identical. Same executables present, same permissions, same creation date, same ownership, same everything. +I made sure that `$PATH` is the same in both environments, I compared the outputs of `bundle env` and `gem environment`, I ran it through `strace`, I dumped the whole environment on both systems and `diff`'d them. +I did not find anything that could be meaningfully different between the 2 systems. And yet, one builds, the other does not. + +That is, until I ran these commands: + +```bash +sudo rm -rf /nix/store/36q0biq46yha892ws5hjcnvlyzc74zsx-ruby2.7.7-image_optim_pack-0.9.1.20230129/lib/ruby/gems/2.7.0/gems/image_optim_pack-0.9.1.20230129/vendor/darwin-x86_64 +sudo rm -rf /nix/store/36q0biq46yha892ws5hjcnvlyzc74zsx-ruby2.7.7-image_optim_pack-0.9.1.20230129/lib/ruby/gems/2.7.0/gems/image_optim_pack-0.9.1.20230129/vendor/linux-i686 +``` + +It builds. +It builds, successfully, correctly, with no errors. +
+ +Then, I went "oh no, this is horrible" and did `nix-store --repair-path`. This does not explain what the problem was, is not *actually* a solution, and I'm not happy with it at all, but it works... kind of. + +With `vendor/darwin-x86_64` and `vendor/linux-i686` gone, the output of `bundle exec image_optim --info` is +``` +Resolved jpegrescan 1a762f62 at /nix/store/c5giaz7fdn0xs67k4i7w62a98vg93416-ruby2.7.7-image_optim-0.31.2/lib/ruby/gems/2.7.0/gems/image_optim-0.31.2/vendor/jpegrescan +pngcrush worker: `pngcrush` not found; please provide proper binary or disable this worker (--no-pngcrush argument or `:pngcrush => false` through options) +pngout worker: `pngout` not found; please provide proper binary or disable this worker (--no-pngout argument or `:pngout => false` through options) +advpng worker: `advpng` not found; please provide proper binary or disable this worker (--no-advpng argument or `:advpng => false` through options) +optipng worker: `optipng` not found; please provide proper binary or disable this worker (--no-optipng argument or `:optipng => false` through options) +pngquant worker: `pngquant` not found; please provide proper binary or disable this worker (--no-pngquant argument or `:pngquant => false` through options) +oxipng worker: `oxipng` not found; please provide proper binary or disable this worker (--no-oxipng argument or `:oxipng => false` through options) +jhead worker: `jhead` not found, `jpegtran` not found; please provide proper binary or disable this worker (--no-jhead argument or `:jhead => false` through options) +jpegoptim worker: `jpegoptim` not found; please provide proper binary or disable this worker (--no-jpegoptim argument or `:jpegoptim => false` through options) +jpegtran worker: `jpegtran` not found; please provide proper binary or disable this worker (--no-jpegtran argument or `:jpegtran => false` through options) +gifsicle worker: `gifsicle` not found; please provide proper binary or disable this worker (--no-gifsicle argument or `:gifsicle => false` through options) +svgo worker: `svgo` not found; please provide proper binary or disable this worker (--no-svgo argument or `:svgo => false` through options) +``` + +Hey, this is better! + +I fixed the rest of the missing binaries like this: +```nix +image_optim_deps = with pkgs; [ + pngout + advancecomp + optipng + pngquant + jhead + jpegoptim + jpeg-archive + libjpeg +]; +``` + +Then added these dependencies to `$PATH`: +```nix +buildPhase = '' + export PATH="${lib.escapeShellArg (lib.makeBinPath image_optim_deps)}":$PATH + + bundle exec jekyll build +''; +``` + +At this point, the derivation builds successfully on both systems. Manually deleting the 2 paths is still horrible, so this is how I ended up with [this](https://github.com/NixOS/nixpkgs/pull/216199) Pull Request to nixpkgs: +```nix +# Clean up vendor binaries for other platforms +image_optim_pack = attrs: + let + vendor_path = "$out/${ruby.gemPath}/gems/image_optim_pack-${attrs.version}/vendor"; + in if builtins.currentSystem == "x86_64-linux" then { + postInstall = '' + rm -rf ${vendor_path}/darwin-x86_64 + rm -rf ${vendor_path}/linux-i686 + ''; + } else if builtins.currentSystem == "i686-linux" then { + postInstall = '' + rm -rf ${vendor_path}/darwin-x86_64 + rm -rf ${vendor_path}/linux-x86_64 + ''; + } else if builtins.currentSystem == "x86_64-darwin" then { + postInstall = '' + rm -rf ${vendor_path}/linux-x86_64 + rm -rf ${vendor_path}/linux-i686 + ''; +} else { }; +``` + +The final package I ended up with is this: +```nix +{ pkgs, lib, ... }: +let + version = "a6476f6fc94f4476f31ebd35687dd54a807d41dd"; + repo = pkgs.fetchgit { + url = "https://git.lain.faith/sorceress/vampysite.git"; + rev = version; + sha256 = "1lr5jqsqik2czgas7ci4fynavhkcw6nfiw1p3lwlxm0ar3y483fk"; + }; + + patched_pkgs = import (builtins.fetchTarball + "https://github.com/AgathaSorceress/nixpkgs/tarball/image-optim-pack-cleanup") { + config = pkgs.config; + }; + + jekyll_env = patched_pkgs.bundlerEnv { + name = "jekyll_env"; + ruby = pkgs.ruby; + + gemdir = "${repo}/."; + }; + image_optim_deps = with pkgs; [ + pngout + advancecomp + optipng + pngquant + jhead + jpegoptim + jpeg-archive + libjpeg + ]; +in pkgs.stdenv.mkDerivation { + inherit version; + name = "vampysite"; + + src = repo; + + buildInputs = with pkgs; [ + jekyll_env + # nokogiri dependencies + zlib + libiconv + libxml2 + libxslt + # jekyll wants a JS runtime + nodejs-slim + ]; + + buildPhase = '' + export PATH="${lib.escapeShellArg (lib.makeBinPath image_optim_deps)}":$PATH + + bundle exec jekyll build + ''; + + installPhase = '' + mkdir -p $out + cp -r _site/* $out/ + ''; +} +``` + +# Deploying + +Now, the fun part. Deploying the website with Nginx is as simple as this: + +```nix +# Nginx +services.nginx = { + enable = true; + + # Use recommended settings + recommendedGzipSettings = true; + recommendedOptimisation = true; + recommendedProxySettings = true; + recommendedTlsSettings = true; + + virtualHosts."technogothic.net" = { + useACMEHost = "technogothic.net"; + forceSSL = true; + root = pkgs.vampysite; + + extraConfig = '' + error_page 404 /404.html; + ''; + }; +}; +``` \ No newline at end of file