ci: run Renovate from asxpi/renovate

Renovate now runs from its own repository for every repo, with the
shared rules in its preset. Extend that preset here and keep only what
is specific to this repo (the Ansible var manager, the Paper JRE, the
runner and nix notes); drop the workflow that ran it for iac alone.

Assisted-by: Claude:opus-5-5

nixos/modules/backup.nix

@@ -0,0 +1,239 @@
+# Nightly restic backup to Hetzner Object Storage, one repository per
+# region, written independently: losing one region or one repository
+# leaves the other complete. The same script and command as the Ansible
+# role restic_backup installs on asxpio-ops:
+#
+#   restic-backup hel1 snapshots
+#   restic-backup fsn1 restore latest --target /tmp/r --include /opt/xmpp
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.asxpio.backup;
+  host = config.networking.hostName;
+
+  # The release the Ansible role pins (restic_version), so both hosts
+  # write their repositories with the same version. nixpkgs 26.05 has an
+  # older one. To bump: change the version, then take the new hash from
+  #   nix store prefetch-file <url>
+  resticVersion = "0.19.1";
+  restic =
+    pkgs.runCommand "restic-${resticVersion}"
+      {
+        src = pkgs.fetchurl {
+          url = "https://github.com/restic/restic/releases/download/v${resticVersion}/restic_${resticVersion}_linux_amd64.bz2";
+          hash = "sha256-9BVBViTcxFLyoCuMM2QXkajG1tO2W7s1Q/z5olFRWFw=";
+        };
+        nativeBuildInputs = [ pkgs.bzip2 ];
+      }
+      ''
+        mkdir -p $out/bin
+        bunzip2 -c $src > $out/bin/restic
+        chmod 755 $out/bin/restic
+      '';
+
+  repositories = {
+    hel1 = "s3:https://hel1.your-objectstorage.com/asxpio-backups-main/restic/${host}";
+    fsn1 = "s3:https://fsn1.your-objectstorage.com/asxpio-backups-repl/restic/${host}";
+  };
+
+  envFile = config.sops.templates."restic-env".path;
+  secret = name: config.sops.placeholder.${name};
+
+  hooks = lib.imap1 (i: hook: ''
+    log "hook ${toString i}/${toString (lib.length cfg.preHooks)}"
+    ( set -e; ${hook} ) || { log "hook failed"; failed=1; }
+  '') cfg.preHooks;
+
+  backup = pkgs.writeShellApplication {
+    name = "restic-backup";
+    runtimeInputs = [
+      restic
+      pkgs.coreutils
+    ]
+    ++ cfg.hookPackages;
+    # No errexit: a failing hook or repository must not stop the others
+    bashOptions = [
+      "nounset"
+      "pipefail"
+    ];
+    text = ''
+      usage() {
+        cat <<'USAGE'
+      restic-backup run                 hooks, then backup + forget to every repository
+      restic-backup init                create the repositories that do not exist yet
+      restic-backup <repo> <args...>    plain restic against one repository, e.g.
+                                        restic-backup hel1 snapshots
+                                        restic-backup fsn1 restore latest --target /tmp/r
+
+      Repositories: ${lib.concatStringsSep ", " (lib.attrNames repositories)}
+      USAGE
+      }
+
+      set -a
+      # shellcheck source=/dev/null
+      . ${envFile}
+      set +a
+      export RESTIC_CACHE_DIR=/var/cache/restic
+      STAGE=${cfg.stageDir}
+
+      declare -A REPOS=(
+      ${lib.concatStrings (lib.mapAttrsToList (name: url: "  [${name}]=\"${url}\"\n") repositories)})
+
+      log() { echo "restic-backup: $*"; }
+
+      case "''${1:-}" in
+      init)
+        for name in "''${!REPOS[@]}"; do
+          if RESTIC_REPOSITORY="''${REPOS[$name]}" restic cat config >/dev/null 2>&1; then
+            log "$name: exists"
+          else
+            RESTIC_REPOSITORY="''${REPOS[$name]}" restic init || exit 1
+            log "$name: created"
+          fi
+        done
+        ;;
+
+      run)
+        failed=0
+        rm -rf "$STAGE"
+        install -d -m 700 "$STAGE"
+        export STAGE
+      ${lib.concatStrings hooks}
+        for name in "''${!REPOS[@]}"; do
+          export RESTIC_REPOSITORY="''${REPOS[$name]}"
+          log "$name: backup"
+          restic backup --host ${host} --one-file-system --exclude-caches \
+            ${lib.concatMapStringsSep " " (e: "--exclude ${lib.escapeShellArg e}") cfg.excludes} \
+            ${lib.escapeShellArgs cfg.paths} "$STAGE" || { failed=1; continue; }
+
+          # Prune rewrites pack files, so costs S3 requests: once a week only
+          prune=()
+          [ "$(date +%a)" = Sun ] && prune=(--prune)
+          log "$name: forget ''${prune[*]}"
+          restic forget --host ${host} --keep-daily 7 --keep-weekly 4 --keep-monthly 6 \
+            "''${prune[@]}" || failed=1
+        done
+
+        rm -rf "$STAGE"
+        exit $failed
+        ;;
+
+      *)
+        [ -n "''${1:-}" ] && [ -n "''${REPOS[$1]:-}" ] || { usage; exit 2; }
+        export RESTIC_REPOSITORY="''${REPOS[$1]}"
+        shift
+        exec restic "$@"
+        ;;
+      esac
+    '';
+  };
+
+  alert = pkgs.writers.writePython3Bin "restic-alert" { doCheck = false; } (
+    builtins.readFile ./restic-alert.py
+  );
+in
+{
+  options.asxpio.backup = {
+    paths = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [
+        "/etc"
+        "/opt"
+      ];
+      description = "What to back up, besides the stage directory.";
+    };
+    excludes = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+    };
+    preHooks = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = ''
+        Shell lines run in order before the backup, with $STAGE set. A
+        failing hook marks the run as failed but the backup still runs.
+      '';
+    };
+    hookPackages = lib.mkOption {
+      type = lib.types.listOf lib.types.package;
+      default = [ ];
+      description = "Programs the hooks call.";
+    };
+    stageDir = lib.mkOption {
+      type = lib.types.str;
+      default = "/var/backups/restic-stage";
+      description = "Dumps written by the hooks land here and are backed up too.";
+    };
+    schedule = lib.mkOption {
+      type = lib.types.str;
+      default = "*-*-* 03:30:00";
+    };
+  };
+
+  config = {
+    sops.secrets = {
+      "restic/password" = { };
+      "s3/access_key" = { };
+      "s3/secret_key" = { };
+      "smtp/user" = { };
+      "smtp/password" = { };
+      "alert/recipient" = { };
+    };
+    sops.templates."restic-env".content = ''
+      RESTIC_PASSWORD=${secret "restic/password"}
+      AWS_ACCESS_KEY_ID=${secret "s3/access_key"}
+      AWS_SECRET_ACCESS_KEY=${secret "s3/secret_key"}
+      ALERT_SMTP_HOST=smtp.fastmail.com
+      ALERT_SMTP_PORT=587
+      ALERT_SMTP_USER=${secret "smtp/user"}
+      ALERT_SMTP_PASSWORD=${secret "smtp/password"}
+      ALERT_RECIPIENT=${secret "alert/recipient"}
+    '';
+
+    environment.systemPackages = [ backup ];
+    systemd.tmpfiles.rules = [
+      "d /var/cache/restic 0700 root root -"
+      "d /var/backups 0755 root root -"
+    ];
+
+    systemd.services.restic-backup = {
+      description = "restic backup to Hetzner Object Storage (${lib.concatStringsSep ", " (lib.attrNames repositories)})";
+      wants = [ "network-online.target" ];
+      after = [
+        "network-online.target"
+        "docker.service"
+      ];
+      onFailure = [ "restic-alert@%n.service" ];
+      serviceConfig = {
+        Type = "oneshot";
+        ExecStart = "${lib.getExe backup} run";
+        Nice = 10;
+        IOSchedulingClass = "idle";
+        TimeoutStartSec = "6h";
+      };
+    };
+    systemd.timers.restic-backup = {
+      description = "Nightly restic backup";
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnCalendar = cfg.schedule;
+        RandomizedDelaySec = "10m";
+        Persistent = true;
+      };
+    };
+
+    # Mails the tail of a failed unit's journal, through Forgejo's SMTP
+    # account. Any unit can use it: OnFailure=restic-alert@%n.service
+    systemd.services."restic-alert@" = {
+      description = "Mail the failure of %i";
+      serviceConfig = {
+        Type = "oneshot";
+        ExecStart = "${lib.getExe alert} %i ${envFile}";
+      };
+    };
+  };
+}

nixos/modules/base.nix

@@ -13,7 +13,7 @@ let
     isNormalUser = true;
     extraGroups = [
       "wheel"
-      "podman"
+      "docker"
     ];
     openssh.authorizedKeys.keys = sshKeys;
   };

nixos/modules/docker.nix

@@ -0,0 +1,25 @@
+# Docker, set up as on the Debian hosts. Declared containers become
+# systemd units named docker-<name>.service.
+{ config, ... }:
+{
+  virtualisation.docker = {
+    enable = true;
+    storageDriver = "overlay2";
+    # For containers nobody declared here, such as CI jobs. Declared
+    # containers log to the journal instead.
+    logDriver = "json-file";
+    daemon.settings.log-opts.max-size = "20m";
+  };
+  virtualisation.oci-containers.backend = "docker";
+
+  # Every image update and CI job leaves an image behind
+  systemd.services.docker-image-prune = {
+    description = "Remove images no container uses";
+    after = [ "docker.service" ];
+    requires = [ "docker.service" ];
+    path = [ config.virtualisation.docker.package ];
+    serviceConfig.Type = "oneshot";
+    script = "docker image prune -af";
+    startAt = "Mon 03:00";
+  };
+}

nixos/modules/restic-alert.py

@@ -0,0 +1,20 @@
+# Mail the tail of a failed unit's journal.
+# Usage: restic-alert <unit> <env file>
+import os, smtplib, socket, subprocess, sys
+from email.message import EmailMessage
+
+unit = sys.argv[1]
+env = dict(l.rstrip("\n").split("=", 1) for l in open(sys.argv[2]) if "=" in l)
+log = subprocess.run(["journalctl", "-u", unit, "-n", "60", "--no-pager", "-o", "cat"],
+                     capture_output=True, text=True).stdout
+
+msg = EmailMessage()
+msg["Subject"] = f"[{socket.gethostname()}] {unit} failed"
+msg["From"] = env["ALERT_SMTP_USER"]
+msg["To"] = env["ALERT_RECIPIENT"]
+msg.set_content(log)
+
+with smtplib.SMTP(env["ALERT_SMTP_HOST"], int(env["ALERT_SMTP_PORT"]), timeout=30) as s:
+    s.starttls()
+    s.login(env["ALERT_SMTP_USER"], env["ALERT_SMTP_PASSWORD"])
+    s.send_message(msg)