dragnpkgs/lib/make-hpc-dist/default.nix

{
  mkShell,
  runCommand,
  stdenvNoCC,
  vmTools,
  writeClosure,
  writeText,

  bash,
  cacert,
  coreutils,
  lix,
  singularity,

  makeSquashFs,

  diskSize ? 1024,
  memSize ? 1024
}:
{
  contents,
  startupScript ? "exec ${bash}/bin/bash -i"
}:
let
  base-container = runCommand "empty.sif.d" {
    buildInputs = [ coreutils ];
  } ''
    mkdir "$out"
    cd "$out"
    mkdir -p proc sys dev nix etc bin usr/bin .singularity.d
    ln -s /etc/sh bin/sh
    ln -s /etc/env usr/bin/env
    ln -s /etc/runscript .singularity.d/runscript
  '';
  container-image = vmTools.runInLinuxVM (
    runCommand "singularity-empty-image" {
      buildInputs = [ base-container singularity ];
    } ''
      export HOME=/tmp
      cp -r "${base-container}" "/tmp/container"
      cd "/tmp"
      find container -type d -exec chmod 755 {} \;

      mkdir -p /var/lib/singularity/mnt/session
      echo "root:x:0:0:System administrator:/root:/bin/sh" > /etc/passwd
      echo > /etc/resolv.conf
      ${singularity}/bin/singularity build "$out/empty.sif" "container/"
    '');

  deps = [ coreutils bash cacert ];

  startupScriptFile = writeText "singularity-startup-script" startupScript;

  shell = stdenvNoCC.mkDerivation {
    name = "shell";
    propagatedBuildInputs = deps ++ contents;
    unpackPhase = "true";
    installPhase = ''
      mkdir -p "$out/bin"
      printf '#!${bash}/bin/bash\n' > "$out/bin/startup.sh"
      export >> "$out/bin/startup.sh"
      cat "${startupScriptFile}" >> "$out/bin/startup.sh"
      chmod +x "$out/bin/startup.sh"
    '';
  };

  base-etc = runCommand "singularity-etc" {
    buildInputs = [ coreutils bash cacert ];
  } ''
    mkdir "$out"
    ln -s "${shell}/bin/startup.sh" "$out/runscript"
    ln -s "${bash}/bin/bash" "$out/sh"
    ln -s "${coreutils}/bin/env" "$out/env"

    mkdir -p "$out/ssl/certs"
    ln -s "${cacert}/etc/ssl/certs/ca-bundle.crt" "$out/ssl/certs/ca-bundle.crt"
    ln -s "${cacert}/etc/ssl/certs/ca-bundle.crt" "$out/ssl/certs/ca-certificates.crt"

    touch "$out/localtime"
    touch "$out/resolv.conf"
  '';


  squashfs = makeSquashFs { filename = "nix-store"; storeContents = [ shell ]; };

  startCommand = writeText "run-container.sh" ''
    #!/usr/bin/env bash
    set -euo pipefail

    module load singularity/3.10.3

    temp_dir="$(mktemp -d)"
    mkdir -p "''${TMPDIR:-/tmp}/empty"
    function __cleanup {
      echo cleaning up
      rsync -r --delete -- "''${TMPDIR:-/tmp}/empty/." "$temp_dir/."
      rmdir "$temp_dir"
      echo done
    }
    trap __cleanup EXIT

    cp -r etc nix-store.squashfs "$temp_dir"

    chmod +w "$temp_dir/etc"
    chmod +w "$temp_dir/etc/resolv.conf"
    chmod +w "$temp_dir/etc/localtime"
    cat /etc/localtime > $temp_dir/etc/localtime
    cat /etc/resolv.conf > $temp_dir/etc/resolv.conf

    singularity run -B "/work:/work,/scratch:/scratch,$temp_dir/nix-store.squashfs:/nix/store:image-src=/,$temp_dir/etc:/etc" --pid --uts --ipc container-base.sif
  '';
in runCommand "hpc-files.d" {} ''
  mkdir "$out"
  cp "${squashfs}" "$out/nix-store.squashfs"
  cp -r "${base-etc}" "$out/etc"
  cp "${container-image}/empty.sif" "$out/container-base.sif"
  cp "${startCommand}" "$out/run-container.sh"
  chmod +x "$out/run-container.sh"
''