ci: run Renovate from asxpi/renovate

Renovate now runs from its own repository for every repo, with the
shared rules in its preset. Extend that preset here and keep only what
is specific to this repo (the Ansible var manager, the Paper JRE, the
runner and nix notes); drop the workflow that ran it for iac alone.

Assisted-by: Claude:opus-5-5

.forgejo/workflows/renovate.yaml

@@ -1,32 +0,0 @@
-name: renovate
-
-# Opens a PR per available version bump (image tags, restic, ...), with
-# the upstream release notes in the PR body. Configuration: renovate.json.
-#
-# Secrets: RENOVATE_TOKEN (Forgejo token of the renovate-bot user, a
-# collaborator with write access) and GH_COM_TOKEN (read-only github.com
-# token, only for fetching release notes; without it PRs have none).
-
-on:
-  schedule:
-    - cron: "17 5 * * *"
-  workflow_dispatch:
-
-jobs:
-  renovate:
-    runs-on: ubuntu-latest
-    container:
-      image: ghcr.io/renovatebot/renovate:44.236.2
-    steps:
-      - name: Run Renovate
-        env:
-          RENOVATE_PLATFORM: forgejo
-          RENOVATE_ENDPOINT: https://git.asxp.io/api/v1
-          RENOVATE_REPOSITORIES: asxpi/iac
-          RENOVATE_TOKEN: ${{ secrets.RENOVATE_TOKEN }}
-          RENOVATE_GITHUB_COM_TOKEN: ${{ secrets.GH_COM_TOKEN }}
-          RENOVATE_GIT_AUTHOR: "Renovate Bot <renovate-bot@noreply.git.asxp.io>"
-          RENOVATE_ONBOARDING: "false"
-          RENOVATE_REQUIRE_CONFIG: required
-          LOG_LEVEL: info
-        run: renovate

nixos/flake.nix

@@ -30,6 +30,8 @@
             sops-nix.nixosModules.sops
             ./modules/base.nix
             ./modules/hetzner.nix
+            ./modules/docker.nix
+            ./modules/backup.nix
           ]
           ++ modules;
         };

nixos/hosts/asxpio-services/containers.nix

@@ -1,123 +1,58 @@
-# Containers run under Podman, one systemd unit each
+# Minecraft and SillyTavern. Their data stays in /opt and is not managed
-# (podman-<name>.service). Game data stays in /opt and is not managed here.
+# here.
-{
+{ config, ... }:
-  config,
-  lib,
-  pkgs,
-  ...
-}:
 let
-  podman = lib.getExe pkgs.podman;
+  # Paper needs Java 21; the tag is a floating JRE line
-  # Time for a Minecraft server to save the world before it is killed
+  javaImage = "docker.io/library/eclipse-temurin:21-jre";
+  paperJar = "paper-1.21.11-54.jar";
+  # Time for the server to save the world before it is killed
   stopTimeout = 120;
 
-  # Nothing that owns data starts on a freshly installed disk: a game
+  # renovate: datasource=docker depName=ghcr.io/sillytavern/sillytavern
-  # server would generate a new world where the restore is about to land.
+  sillytavernVersion = "1.19.0";
-  # After restoring /opt, `touch /opt/.restored` and start the units.
-  needsData.unitConfig.ConditionPathExists = "/opt/.restored";
-
-  gameServer = needsData // {
-    serviceConfig = {
-      TimeoutStopSec = lib.mkForce (stopTimeout + 30);
-      # A JVM stopped by SIGTERM exits 143 after saving; that is a clean stop
-      SuccessExitStatus = [ 143 ];
-    };
-  };
 in
 {
-  virtualisation.podman = {
+  # Paper server; the jar and the world live in /opt/minecraft. Not an
-    enable = true;
+  # oci-containers unit: the console needs a TTY and an open stdin, and
-    dockerCompat = true; # `docker` runs podman
+  # `docker run -it` refuses to start without a terminal. The container
-    dockerSocket.enable = true; # /run/docker.sock, used by the CI runner
+  # is created first and then attached to, output only.
-    defaultNetwork.settings.dns_enabled = true;
+  #
-    autoPrune = {
+  #   docker attach minecraft        console; leave with Ctrl-p Ctrl-q
-      enable = true;
+  systemd.services.docker-minecraft = {
-      dates = "Mon 03:00";
+    description = "Minecraft (Paper)";
-      flags = [ "--all" ];
+    wantedBy = [ "multi-user.target" ];
-    };
+    wants = [ "network-online.target" ];
-  };
+    # requires: Docker restarting stops the server first, with time to save
-  virtualisation.containers.registries.search = [ "docker.io" ];
+    requires = [ "docker.service" ];
-  virtualisation.oci-containers.backend = "podman";
+    after = [
-
+      "docker.service"
-  sops.secrets."minecraft_fossils/env" = { };
+      "network-online.target"
-
-  virtualisation.oci-containers.containers = {
-    # Paper server; the jar and the world live in /opt/minecraft
-    minecraft = {
-      image = "docker.io/library/eclipse-temurin:21-jre";
-      workdir = "/data";
-      cmd = [
-        "java"
-        "-Xmx4G"
-        "-Xms2G"
-        "-jar"
-        "paper-1.21.11-54.jar"
-        "nogui"
-      ];
-      ports = [
-        "25565:25565"
-        "19132:19132/udp"
-      ];
-      volumes = [ "/opt/minecraft:/data" ];
-      extraOptions = [
-        "--tty"
-        "--interactive"
-        "--stop-timeout=${toString stopTimeout}"
     ];
+    unitConfig.ConditionPathExists = "/opt/.restored";
+    path = [ config.virtualisation.docker.package ];
+    preStart = ''
+      docker rm -f minecraft || true
+      docker create --name minecraft --tty --interactive \
+        --stop-timeout ${toString stopTimeout} \
+        --workdir /data --volume /opt/minecraft:/data \
+        --publish 25565:25565 --publish 19132:19132/udp \
+        ${javaImage} java -Xmx4G -Xms2G -jar ${paperJar} nogui
+    '';
+    script = "exec docker start --attach minecraft";
+    preStop = "docker stop minecraft || true";
+    postStop = "docker rm -f minecraft || true";
+    serviceConfig = {
+      TimeoutStartSec = 0; # the first start pulls the image
+      TimeoutStopSec = stopTimeout + 30;
+      # A JVM stopped by SIGTERM exits 143 after saving; that is a clean stop
+      SuccessExitStatus = [ 143 ];
+      Restart = "on-failure";
     };
-
-    minecraft-fossils = {
-      image = "docker.io/itzg/minecraft-server:java21";
-      environment = {
-        EULA = "TRUE";
-        TYPE = "FABRIC";
-        VERSION = "1.20.1";
-        FABRIC_LOADER_VERSION = "0.18.4";
-        MEMORY = "8G";
-        ONLINE_MODE = "FALSE";
-        ENFORCE_WHITELIST = "TRUE";
-        ENFORCE_SECURE_PROFILE = "FALSE";
-        WHITELIST_FILE = "/data/whitelist.json";
-        OPS_FILE = "/data/ops.json";
-        ENABLE_RCON = "TRUE";
-        MOTD = "[mc.czsk.dev] Fossils Archaeology Revival: Genesis";
-        USE_AIKAR_FLAGS = "TRUE";
-      };
-      environmentFiles = [ config.sops.secrets."minecraft_fossils/env".path ];
-      ports = [ "25566:25565" ];
-      volumes = [ "/opt/minecraft-fossils/data:/data" ];
-      extraOptions = [
-        "--tty"
-        "--interactive"
-        "--network=fossils"
-        "--stop-timeout=${toString stopTimeout}"
-      ];
-    };
-
-    # Copies whitelist, ops and ban lists from the Paper server
-    minecraft-fossils-wlsync = {
-      image = "docker.io/library/alpine:3";
-      dependsOn = [ "minecraft-fossils" ];
-      environment = {
-        RCON_HOST = "minecraft-fossils";
-        INTERVAL = "15";
-      };
-      environmentFiles = [ config.sops.secrets."minecraft_fossils/env".path ];
-      volumes = [
-        "/opt/minecraft:/src:ro"
-        "/opt/minecraft-fossils/data:/dst"
-        "${./wlsync.sh}:/wlsync.sh:ro"
-      ];
-      cmd = [
-        "sh"
-        "/wlsync.sh"
-      ];
-      extraOptions = [ "--network=fossils" ];
   };
 
   # Reachable on the WireGuard address only
-    sillytavern = {
+  virtualisation.oci-containers.containers.sillytavern = {
-      image = "ghcr.io/sillytavern/sillytavern:1.19.0";
+    image = "ghcr.io/sillytavern/sillytavern:${sillytavernVersion}";
     ports = [ "10.20.0.20:8000:8000" ];
     volumes = [
       "/opt/sillytavern/config:/home/node/app/config"
@@ -125,59 +60,22 @@ in
       "/opt/sillytavern/plugins:/home/node/app/plugins"
     ];
   };
-  };
-
-  systemd.services = {
-    # Network shared by the Fossils server and its sidecar
-    podman-network-fossils = {
-      description = "Podman network fossils";
-      wantedBy = [ "multi-user.target" ];
-      before = [
-        "podman-minecraft-fossils.service"
-        "podman-minecraft-fossils-wlsync.service"
-      ];
-      requiredBy = [
-        "podman-minecraft-fossils.service"
-        "podman-minecraft-fossils-wlsync.service"
-      ];
-      serviceConfig = {
-        Type = "oneshot";
-        RemainAfterExit = true;
-      };
-      script = "${podman} network exists fossils || ${podman} network create fossils";
-    };
-
-    podman-minecraft = gameServer;
-    podman-minecraft-fossils = gameServer;
-    podman-minecraft-fossils-wlsync = needsData;
 
   # The WireGuard address must exist before a port can be bound to it
-    podman-sillytavern = needsData // {
+  systemd.services.docker-sillytavern = {
+    unitConfig.ConditionPathExists = "/opt/.restored";
     after = [ "wireguard-asxpio-services.service" ];
     requires = [ "wireguard-asxpio-services.service" ];
   };
 
-    minecraft-fossils-restart = {
+  # Published ports reach the containers without passing these rules over
-      description = "Daily restart of minecraft-fossils";
+  # IPv4. Over IPv6 Docker's proxy listens on the host, so they are needed.
-      serviceConfig.Type = "oneshot";
+  networking.firewall.allowedTCPPorts = [ 25565 ];
-      script = "systemctl restart podman-minecraft-fossils.service";
+  networking.firewall.allowedUDPPorts = [ 19132 ];
-    };
-  };
-
-  systemd.timers.minecraft-fossils-restart = {
-    description = "Restart minecraft-fossils daily at 08:00 Europe/Tallinn";
-    wantedBy = [ "timers.target" ];
-    timerConfig = {
-      OnCalendar = "*-*-* 08:00:00 Europe/Tallinn";
-      Persistent = true;
-    };
-  };
 
   systemd.tmpfiles.rules = [
     "d /opt 0755 root root -"
     "d /opt/minecraft 0755 root root -"
-    "d /opt/minecraft-fossils 0755 asxpi users -"
-    "d /opt/minecraft-fossils/data 0755 asxpi users -"
     "d /opt/sillytavern/config 0755 root root -"
     "d /opt/sillytavern/data 0755 root root -"
     "d /opt/sillytavern/plugins 0755 root root -"

nixos/hosts/asxpio-services/default.nix

@@ -1,9 +1,12 @@
-# asxpio-services: CI runner, Minecraft, SillyTavern.
+# asxpio-services: CI runner, Minecraft, XMPP, Xray, SillyTavern.
 { config, lib, ... }:
 {
   imports = [
     ./containers.nix
+    ./replica.nix
     ./runner.nix
+    ./xmpp.nix
+    ./xray.nix
   ];
 
   networking.hostName = lib.mkDefault "asxpio-services";
@@ -59,27 +62,26 @@
   };
 
   # --- Firewall -----------------------------------------------------------
-  # The Hetzner firewall is the outer layer. Published container ports are
+  # The Hetzner firewall is the outer layer. Each service opens its own
-  # forwarded by Podman and do not pass through these input rules.
+  # ports next to its definition.
-  networking.firewall = {
+  networking.firewall.interfaces = {
-    allowedTCPPorts = [
-      25565
-      25566
-    ];
-    allowedUDPPorts = [ 19132 ];
-    interfaces = {
     asxpio-services.allowedTCPPorts = [
       22
       8000
     ];
     enp7s0.allowedTCPPorts = [ 22 ];
-      # CI job containers reach the runner's cache server on the host
+  };
-      "podman+".allowedTCPPortRanges = [
+
-        {
+  # --- Backup -------------------------------------------------------------
-          from = 1024;
+  # The Paper world is copied live (RCON is off, so there is no save-off);
-          to = 65535;
+  # its autosave keeps that close enough. What Paper downloads again is
-        }
+  # left out.
+  asxpio.backup = {
+    schedule = "*-*-* 04:15:00";
+    excludes = [
+      "/opt/minecraft/cache"
+      "/opt/minecraft/libraries"
+      "/opt/minecraft/versions"
     ];
   };
-  };
 }

nixos/hosts/asxpio-services/replica.nix

@@ -0,0 +1,93 @@
+# Daily one-way replica of the hel1 buckets into their fsn1 twins.
+# restic/ is excluded: restic writes its own, independent fsn1 repository.
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  pairs = [
+    {
+      src = "asxpio-backups-main";
+      dst = "asxpio-backups-repl";
+      exclude = [ "restic/**" ];
+    }
+    {
+      src = "asxpio-terraform-main";
+      dst = "asxpio-terraform-repl";
+    }
+    {
+      src = "k8s-asxpio-main";
+      dst = "k8s-asxpio-repl";
+    }
+  ];
+
+  # A run that would delete more than this stops and fails (and mails):
+  # a mistake on the hel1 side must not empty the replica the next morning
+  maxDelete = 1000;
+
+  secret = name: config.sops.placeholder.${name};
+  remote = name: region: extra: ''
+    RCLONE_CONFIG_${name}_TYPE=s3
+    RCLONE_CONFIG_${name}_PROVIDER=Other
+    RCLONE_CONFIG_${name}_ENDPOINT=https://${region}.your-objectstorage.com
+    RCLONE_CONFIG_${name}_REGION=${region}
+    RCLONE_CONFIG_${name}_ACCESS_KEY_ID=${secret "s3/access_key"}
+    RCLONE_CONFIG_${name}_SECRET_ACCESS_KEY=${secret "s3/secret_key"}
+    ${extra}'';
+
+  replicate = pkgs.writeShellApplication {
+    name = "s3-replicate";
+    runtimeInputs = [ pkgs.rclone ];
+    # No errexit: a failing pair does not stop the others, but fails the run
+    bashOptions = [
+      "nounset"
+      "pipefail"
+    ];
+    text = ''
+      failed=0
+    ''
+    + lib.concatMapStrings (p: ''
+      echo "s3-replicate: ${p.src} -> ${p.dst}"
+      rclone sync hel1:${p.src} fsn1:${p.dst} \
+        --fast-list --checkers 16 --transfers 8 --max-delete ${toString maxDelete} \
+        ${lib.concatMapStringsSep " " (e: "--exclude ${lib.escapeShellArg e}") (p.exclude or [ ])} \
+        --stats-one-line -v || failed=1
+    '') pairs
+    + ''
+      exit $failed
+    '';
+  };
+in
+{
+  # The object storage remotes hel1: and fsn1:, as rclone environment
+  # variables. The migration's restore script reads them too.
+  sops.templates."s3-replicate-env".content =
+    remote "HEL1" "hel1" ""
+    + remote "FSN1" "fsn1" ''
+      RCLONE_CONFIG_FSN1_NO_CHECK_BUCKET=true
+    '';
+
+  systemd.services.s3-replicate = {
+    description = "Replicate the hel1 buckets to fsn1";
+    wants = [ "network-online.target" ];
+    after = [ "network-online.target" ];
+    onFailure = [ "restic-alert@%n.service" ];
+    serviceConfig = {
+      Type = "oneshot";
+      ExecStart = lib.getExe replicate;
+      EnvironmentFile = config.sops.templates."s3-replicate-env".path;
+      TimeoutStartSec = "12h";
+    };
+  };
+  systemd.timers.s3-replicate = {
+    description = "Daily hel1 -> fsn1 bucket replica";
+    wantedBy = [ "timers.target" ];
+    timerConfig = {
+      OnCalendar = "*-*-* 06:00:00";
+      RandomizedDelaySec = "15m";
+      Persistent = true;
+    };
+  };
+}

nixos/hosts/asxpio-services/runner.nix

@@ -1,13 +1,22 @@
 # Forgejo Actions runner. It runs as a container and starts job containers
-# through Podman's Docker-compatible socket. Its registration lives in
+# through the Docker socket. Its registration lives in
 # /opt/forgejo-runner/data/.runner and is state, not configuration.
+#
+# Deploying a change to it restarts the runner under the job that deploys:
+# the change lands, that job shows as cancelled, and the next run confirms
+# the result.
 { pkgs, ... }:
 let
+  # renovate: datasource=docker depName=code.forgejo.org/forgejo/runner
+  runnerVersion = "13";
+
   # Image behind the nix-latest label. Jobs mount a nix-store volume over
   # /nix, seeded from this image on first use; the image's /bin/sh and
   # /etc/nix/nix.conf link into that store, so a volume from another
-  # version breaks them. Bumping this also switches to a fresh volume.
+  # version breaks them. Bumping this also switches to a fresh volume;
-  nixVersion = "2.35.3";
+  # remove the old one afterwards: docker volume rm nix-store-<old version>
+  # renovate: datasource=docker depName=nixos/nix
+  nixVersion = "2.35.6";
 
   settings = {
     log = {
@@ -65,8 +74,8 @@ let
 in
 {
   virtualisation.oci-containers.containers.forgejo-runner = {
-    image = "code.forgejo.org/forgejo/runner:11";
+    image = "code.forgejo.org/forgejo/runner:${runnerVersion}";
-    user = "0:0"; # needs the container socket
+    user = "0:0"; # needs the Docker socket
     cmd = [
       "forgejo-runner"
       "daemon"
@@ -77,16 +86,28 @@ in
     volumes = [
       "/opt/forgejo-runner/data:/data"
       "${configFile}:/config.yaml:ro"
-      "/run/podman/podman.sock:/var/run/docker.sock"
+      "/var/run/docker.sock:/var/run/docker.sock"
     ];
     extraOptions = [ "--network=host" ];
   };
 
-  systemd.services.podman-forgejo-runner = {
-    after = [ "podman.socket" ];
-    requires = [ "podman.socket" ];
   # Without its registration the runner only crash-loops
-    unitConfig.ConditionPathExists = "/opt/forgejo-runner/data/.runner";
+  systemd.services.docker-forgejo-runner.unitConfig.ConditionPathExists =
+    "/opt/forgejo-runner/data/.runner";
+
+  # CI job containers reach the runner's cache server on the host
+  networking.firewall.interfaces =
+    let
+      highPorts.allowedTCPPortRanges = [
+        {
+          from = 1024;
+          to = 65535;
+        }
+      ];
+    in
+    {
+      docker0 = highPorts;
+      "br-+" = highPorts;
     };
 
   systemd.tmpfiles.rules = [ "d /opt/forgejo-runner/data 0777 root root -" ];

nixos/hosts/asxpio-services/wlsync.sh

@@ -1,35 +0,0 @@
-#!/bin/sh
-# Mirrors whitelist, ops and ban lists from the Paper server into the
-# Fossils server and tells it to reload them over RCON.
-apk add --no-cache python3 >/dev/null
-cat > /rcon.py <<'PY'
-import socket, struct, sys, os
-host, port, pw, cmd = os.environ["RCON_HOST"], 25575, os.environ["RCON_PASSWORD"], sys.argv[1]
-def pkt(i, t, b): p = struct.pack("<ii", i, t) + b.encode() + b"\x00\x00"; return struct.pack("<i", len(p)) + p
-def recv(s):
-    ln = struct.unpack("<i", s.recv(4))[0]; d = b""
-    while len(d) < ln: d += s.recv(ln - len(d))
-    return struct.unpack("<ii", d[:8])[0], d[8:-2].decode(errors="replace")
-s = socket.create_connection((host, port), timeout=5)
-s.send(pkt(1, 3, pw))
-rid, _ = recv(s)
-if rid == -1: print("auth failed"); sys.exit(1)
-s.send(pkt(2, 2, cmd))
-_, out = recv(s); print(out)
-PY
-echo "[wlsync] started, interval=${INTERVAL}s"
-while true; do
-  for f in whitelist.json ops.json banned-players.json banned-ips.json; do
-    if [ -f /src/$f ] && ! cmp -s /src/$f /dst/$f 2>/dev/null; then
-      cp /src/$f /dst/$f
-      echo "[wlsync] copied $f"
-      case $f in
-        whitelist.json) CMD="whitelist reload" ;;
-        ops.json)       CMD="reload" ;;
-        banned-players.json|banned-ips.json) CMD="kick @a Ban list updated" ;;
-      esac
-      python3 /rcon.py "$CMD" 2>&1 | sed 's/^/[wlsync] rcon: /' || echo "[wlsync] rcon failed"
-    fi
-  done
-  sleep $INTERVAL
-done

nixos/hosts/asxpio-services/xmpp.nix

@@ -0,0 +1,134 @@
+# another.im XMPP server for asxp.io: ejabberd, apushd (push), tackygram
+# (Telegram gateway). After compose.yaml and compose.tackygram.yaml of
+# https://dev.another.im/another-im/anotherim-server, CERTS=files.
+#
+# Secrets and keys are host data in /opt/xmpp/state, like a game world,
+# and are covered by the restic backup:
+#   secrets.env          ejabberd secrets
+#   certs/server.pem     TLS certificate (asxp.io, conference., xmpp.)
+#   apushd/              push settings and VAPID key
+#   tackygram/{env,key}  gateway credentials and master key
+# Data: Docker volumes xmpp_database, xmpp_tackygram.
+{ config, pkgs, ... }:
+let
+  # renovate: datasource=docker depName=dev.another.im/another-im/anotherim-server
+  ejabberdVersion = "2026.11.07";
+  # renovate: datasource=docker depName=dev.another.im/another-im/apushd
+  apushdVersion = "0.2";
+  # renovate: datasource=docker depName=dev.another.im/another-im/tackygram
+  tackygramVersion = "0.4.3";
+
+  domain = "asxp.io";
+  publicIp = "46.62.228.216";
+  state = "/opt/xmpp/state";
+
+  tcpPorts = [
+    5222 # clients, STARTTLS
+    5223 # clients, direct TLS
+    5269 # other servers
+    5443 # WebSocket, host-meta
+    8448 # Matrix federation
+    3478 # STUN and TURN
+    7777 # SOCKS5 bytestreams proxy
+  ];
+  turnRelay = {
+    from = 49160;
+    to = 49199;
+  };
+  gatewayCalls = {
+    from = 50000;
+    to = 50099;
+  };
+  range = r: "${toString r.from}-${toString r.to}";
+
+  needsData.unitConfig.ConditionPathExists = "/opt/.restored";
+in
+{
+  virtualisation.oci-containers.containers = {
+    xmpp-ejabberd = {
+      image = "dev.another.im/another-im/anotherim-server:${ejabberdVersion}";
+      environment = {
+        EJABBERD_MACRO_XMPP_DOMAIN = domain;
+        EJABBERD_MACRO_TURN_IP = publicIp;
+        EJABBERD_MACRO_POW_DIFFICULTY = "6";
+        EJABBERD_MACRO_REGISTRATION_TIMEOUT = "600";
+        EJABBERD_MACRO_UPLOAD_URL = "https://hel1.your-objectstorage.com/asxpio-xmpp-public/";
+        EJABBERD_MACRO_ACME_URL = "https://acme-v02.api.letsencrypt.org/directory";
+      };
+      environmentFiles = [ "${state}/secrets.env" ];
+      ports =
+        map (p: "${toString p}:${toString p}") tcpPorts
+        ++ [
+          "3478:3478/udp"
+          "${range turnRelay}:${range turnRelay}/udp"
+          "${range gatewayCalls}:${range gatewayCalls}/udp"
+        ];
+      volumes = [
+        "xmpp_database:/opt/ejabberd/database"
+        "xmpp_logs:/opt/ejabberd/logs"
+        "${state}/certs:/opt/ejabberd/conf/certs:ro"
+      ];
+    };
+
+    # Push App Server and Telegram gateway share ejabberd's network
+    # namespace: they reach its component port on 127.0.0.1 only. They
+    # restart with it, since the namespace is gone when it stops.
+    xmpp-apushd = {
+      image = "dev.another.im/another-im/apushd:${apushdVersion}";
+      dependsOn = [ "xmpp-ejabberd" ];
+      networks = [ "container:xmpp-ejabberd" ];
+      volumes = [ "${state}/apushd:/etc/apushd:ro" ];
+    };
+
+    xmpp-tackygram = {
+      image = "dev.another.im/another-im/tackygram:${tackygramVersion}";
+      dependsOn = [ "xmpp-ejabberd" ];
+      networks = [ "container:xmpp-ejabberd" ];
+      environmentFiles = [ "${state}/tackygram/env" ];
+      environment = {
+        CREDENTIALS_DIRECTORY = "/run/credentials";
+        TACKYGRAM_CALL_IP = publicIp;
+        TACKYGRAM_CALL_PORTS = range gatewayCalls;
+      };
+      volumes = [
+        "xmpp_tackygram:/var/lib/tackygram"
+        "${state}/tackygram/key:/run/credentials/gateway-key:ro"
+      ];
+    };
+  };
+
+  systemd.services = {
+    docker-xmpp-ejabberd = needsData;
+    docker-xmpp-apushd = needsData;
+    docker-xmpp-tackygram = needsData;
+  };
+
+  # For IPv6; see containers.nix
+  networking.firewall = {
+    allowedTCPPorts = tcpPorts;
+    allowedUDPPorts = [ 3478 ];
+    allowedUDPPortRanges = [
+      turnRelay
+      gatewayCalls
+    ];
+  };
+
+  systemd.tmpfiles.rules = [ "d ${state} 0700 root root -" ];
+
+  # ejabberd's Mnesia through its own backup command, its SQLite through
+  # the online backup API
+  asxpio.backup = {
+    paths = [
+      "/var/lib/docker/volumes/xmpp_database"
+      "/var/lib/docker/volumes/xmpp_tackygram"
+    ];
+    hookPackages = [
+      config.virtualisation.docker.package
+      pkgs.sqlite
+    ];
+    preHooks = [
+      "docker exec xmpp-ejabberd ejabberdctl backup /opt/ejabberd/database/nightly.backup"
+      ''sqlite3 /var/lib/docker/volumes/xmpp_database/_data/ejabberd.sqlite ".backup '$STAGE/ejabberd.sqlite'"''
+    ];
+  };
+}

nixos/hosts/asxpio-services/xray.nix

@@ -0,0 +1,32 @@
+# Xray VLESS Reality proxy on :443, host network. Root in the container
+# only to read the 0400 config and bind :443; every other capability is
+# dropped.
+{ config, ... }:
+let
+  # renovate: datasource=docker depName=ghcr.io/xtls/xray-core
+  xrayVersion = "26.11.9";
+in
+{
+  # The whole config.json; a new one restarts the container
+  sops.secrets."xray/config".restartUnits = [ "docker-xray.service" ];
+
+  virtualisation.oci-containers.containers.xray = {
+    image = "ghcr.io/xtls/xray-core:${xrayVersion}";
+    cmd = [
+      "run"
+      "-c"
+      "/etc/xray/config.json"
+    ];
+    user = "0";
+    volumes = [ "${config.sops.secrets."xray/config".path}:/etc/xray/config.json:ro" ];
+    extraOptions = [
+      "--network=host"
+      "--cap-drop=ALL"
+      "--cap-add=NET_BIND_SERVICE"
+      "--security-opt=no-new-privileges:true"
+      "--read-only"
+    ];
+  };
+
+  networking.firewall.allowedTCPPorts = [ 443 ];
+}

nixos/hosts/rehearsal.nix

@@ -1,6 +1,9 @@
 # Overrides that turn asxpio-services into a throwaway test machine:
 # plain DHCP, SSH from anywhere, no WireGuard, dummy secrets.
 { lib, ... }:
+let
+  manual.autoStart = lib.mkForce false;
+in
 {
   networking.hostName = "rehearsal";
   sops.defaultSopsFile = ../secrets/rehearsal.yaml;
@@ -13,11 +16,26 @@
   };
 
   networking.wireguard.interfaces = lib.mkForce { };
-  systemd.services.podman-sillytavern = {
+  systemd.services.docker-sillytavern = {
     after = lib.mkForce [ ];
     requires = lib.mkForce [ ];
   };
-  virtualisation.oci-containers.containers.sillytavern.ports = lib.mkForce [ "127.0.0.1:8000:8000" ];
+  virtualisation.oci-containers.containers = {
+    sillytavern.ports = lib.mkForce [ "127.0.0.1:8000:8000" ];
+
+    # With production data restored these would act as production: a
+    # second asxp.io XMPP server and Telegram session, a runner taking
+    # real CI jobs. Start them by hand, and only with test data.
+    xmpp-ejabberd = manual;
+    xmpp-apushd = manual;
+    xmpp-tackygram = manual;
+    forgejo-runner = manual;
+    xray = manual; # its config is a dummy
+  };
+
+  # No backups of, or from, a test machine
+  systemd.timers.restic-backup.enable = false;
+  systemd.timers.s3-replicate.enable = false;
 
   services.openssh.openFirewall = lib.mkForce true;
   services.openssh.settings.AllowUsers = [ "root" ];

nixos/modules/backup.nix

@@ -0,0 +1,239 @@
+# Nightly restic backup to Hetzner Object Storage, one repository per
+# region, written independently: losing one region or one repository
+# leaves the other complete. The same script and command as the Ansible
+# role restic_backup installs on asxpio-ops:
+#
+#   restic-backup hel1 snapshots
+#   restic-backup fsn1 restore latest --target /tmp/r --include /opt/xmpp
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.asxpio.backup;
+  host = config.networking.hostName;
+
+  # The release the Ansible role pins (restic_version), so both hosts
+  # write their repositories with the same version. nixpkgs 26.05 has an
+  # older one. To bump: change the version, then take the new hash from
+  #   nix store prefetch-file <url>
+  resticVersion = "0.19.1";
+  restic =
+    pkgs.runCommand "restic-${resticVersion}"
+      {
+        src = pkgs.fetchurl {
+          url = "https://github.com/restic/restic/releases/download/v${resticVersion}/restic_${resticVersion}_linux_amd64.bz2";
+          hash = "sha256-9BVBViTcxFLyoCuMM2QXkajG1tO2W7s1Q/z5olFRWFw=";
+        };
+        nativeBuildInputs = [ pkgs.bzip2 ];
+      }
+      ''
+        mkdir -p $out/bin
+        bunzip2 -c $src > $out/bin/restic
+        chmod 755 $out/bin/restic
+      '';
+
+  repositories = {
+    hel1 = "s3:https://hel1.your-objectstorage.com/asxpio-backups-main/restic/${host}";
+    fsn1 = "s3:https://fsn1.your-objectstorage.com/asxpio-backups-repl/restic/${host}";
+  };
+
+  envFile = config.sops.templates."restic-env".path;
+  secret = name: config.sops.placeholder.${name};
+
+  hooks = lib.imap1 (i: hook: ''
+    log "hook ${toString i}/${toString (lib.length cfg.preHooks)}"
+    ( set -e; ${hook} ) || { log "hook failed"; failed=1; }
+  '') cfg.preHooks;
+
+  backup = pkgs.writeShellApplication {
+    name = "restic-backup";
+    runtimeInputs = [
+      restic
+      pkgs.coreutils
+    ]
+    ++ cfg.hookPackages;
+    # No errexit: a failing hook or repository must not stop the others
+    bashOptions = [
+      "nounset"
+      "pipefail"
+    ];
+    text = ''
+      usage() {
+        cat <<'USAGE'
+      restic-backup run                 hooks, then backup + forget to every repository
+      restic-backup init                create the repositories that do not exist yet
+      restic-backup <repo> <args...>    plain restic against one repository, e.g.
+                                        restic-backup hel1 snapshots
+                                        restic-backup fsn1 restore latest --target /tmp/r
+
+      Repositories: ${lib.concatStringsSep ", " (lib.attrNames repositories)}
+      USAGE
+      }
+
+      set -a
+      # shellcheck source=/dev/null
+      . ${envFile}
+      set +a
+      export RESTIC_CACHE_DIR=/var/cache/restic
+      STAGE=${cfg.stageDir}
+
+      declare -A REPOS=(
+      ${lib.concatStrings (lib.mapAttrsToList (name: url: "  [${name}]=\"${url}\"\n") repositories)})
+
+      log() { echo "restic-backup: $*"; }
+
+      case "''${1:-}" in
+      init)
+        for name in "''${!REPOS[@]}"; do
+          if RESTIC_REPOSITORY="''${REPOS[$name]}" restic cat config >/dev/null 2>&1; then
+            log "$name: exists"
+          else
+            RESTIC_REPOSITORY="''${REPOS[$name]}" restic init || exit 1
+            log "$name: created"
+          fi
+        done
+        ;;
+
+      run)
+        failed=0
+        rm -rf "$STAGE"
+        install -d -m 700 "$STAGE"
+        export STAGE
+      ${lib.concatStrings hooks}
+        for name in "''${!REPOS[@]}"; do
+          export RESTIC_REPOSITORY="''${REPOS[$name]}"
+          log "$name: backup"
+          restic backup --host ${host} --one-file-system --exclude-caches \
+            ${lib.concatMapStringsSep " " (e: "--exclude ${lib.escapeShellArg e}") cfg.excludes} \
+            ${lib.escapeShellArgs cfg.paths} "$STAGE" || { failed=1; continue; }
+
+          # Prune rewrites pack files, so costs S3 requests: once a week only
+          prune=()
+          [ "$(date +%a)" = Sun ] && prune=(--prune)
+          log "$name: forget ''${prune[*]}"
+          restic forget --host ${host} --keep-daily 7 --keep-weekly 4 --keep-monthly 6 \
+            "''${prune[@]}" || failed=1
+        done
+
+        rm -rf "$STAGE"
+        exit $failed
+        ;;
+
+      *)
+        [ -n "''${1:-}" ] && [ -n "''${REPOS[$1]:-}" ] || { usage; exit 2; }
+        export RESTIC_REPOSITORY="''${REPOS[$1]}"
+        shift
+        exec restic "$@"
+        ;;
+      esac
+    '';
+  };
+
+  alert = pkgs.writers.writePython3Bin "restic-alert" { doCheck = false; } (
+    builtins.readFile ./restic-alert.py
+  );
+in
+{
+  options.asxpio.backup = {
+    paths = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [
+        "/etc"
+        "/opt"
+      ];
+      description = "What to back up, besides the stage directory.";
+    };
+    excludes = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+    };
+    preHooks = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = ''
+        Shell lines run in order before the backup, with $STAGE set. A
+        failing hook marks the run as failed but the backup still runs.
+      '';
+    };
+    hookPackages = lib.mkOption {
+      type = lib.types.listOf lib.types.package;
+      default = [ ];
+      description = "Programs the hooks call.";
+    };
+    stageDir = lib.mkOption {
+      type = lib.types.str;
+      default = "/var/backups/restic-stage";
+      description = "Dumps written by the hooks land here and are backed up too.";
+    };
+    schedule = lib.mkOption {
+      type = lib.types.str;
+      default = "*-*-* 03:30:00";
+    };
+  };
+
+  config = {
+    sops.secrets = {
+      "restic/password" = { };
+      "s3/access_key" = { };
+      "s3/secret_key" = { };
+      "smtp/user" = { };
+      "smtp/password" = { };
+      "alert/recipient" = { };
+    };
+    sops.templates."restic-env".content = ''
+      RESTIC_PASSWORD=${secret "restic/password"}
+      AWS_ACCESS_KEY_ID=${secret "s3/access_key"}
+      AWS_SECRET_ACCESS_KEY=${secret "s3/secret_key"}
+      ALERT_SMTP_HOST=smtp.fastmail.com
+      ALERT_SMTP_PORT=587
+      ALERT_SMTP_USER=${secret "smtp/user"}
+      ALERT_SMTP_PASSWORD=${secret "smtp/password"}
+      ALERT_RECIPIENT=${secret "alert/recipient"}
+    '';
+
+    environment.systemPackages = [ backup ];
+    systemd.tmpfiles.rules = [
+      "d /var/cache/restic 0700 root root -"
+      "d /var/backups 0755 root root -"
+    ];
+
+    systemd.services.restic-backup = {
+      description = "restic backup to Hetzner Object Storage (${lib.concatStringsSep ", " (lib.attrNames repositories)})";
+      wants = [ "network-online.target" ];
+      after = [
+        "network-online.target"
+        "docker.service"
+      ];
+      onFailure = [ "restic-alert@%n.service" ];
+      serviceConfig = {
+        Type = "oneshot";
+        ExecStart = "${lib.getExe backup} run";
+        Nice = 10;
+        IOSchedulingClass = "idle";
+        TimeoutStartSec = "6h";
+      };
+    };
+    systemd.timers.restic-backup = {
+      description = "Nightly restic backup";
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnCalendar = cfg.schedule;
+        RandomizedDelaySec = "10m";
+        Persistent = true;
+      };
+    };
+
+    # Mails the tail of a failed unit's journal, through Forgejo's SMTP
+    # account. Any unit can use it: OnFailure=restic-alert@%n.service
+    systemd.services."restic-alert@" = {
+      description = "Mail the failure of %i";
+      serviceConfig = {
+        Type = "oneshot";
+        ExecStart = "${lib.getExe alert} %i ${envFile}";
+      };
+    };
+  };
+}

nixos/modules/base.nix

@@ -13,7 +13,7 @@ let
     isNormalUser = true;
     extraGroups = [
       "wheel"
-      "podman"
+      "docker"
     ];
     openssh.authorizedKeys.keys = sshKeys;
   };

nixos/modules/docker.nix

@@ -0,0 +1,25 @@
+# Docker, set up as on the Debian hosts. Declared containers become
+# systemd units named docker-<name>.service.
+{ config, ... }:
+{
+  virtualisation.docker = {
+    enable = true;
+    storageDriver = "overlay2";
+    # For containers nobody declared here, such as CI jobs. Declared
+    # containers log to the journal instead.
+    logDriver = "json-file";
+    daemon.settings.log-opts.max-size = "20m";
+  };
+  virtualisation.oci-containers.backend = "docker";
+
+  # Every image update and CI job leaves an image behind
+  systemd.services.docker-image-prune = {
+    description = "Remove images no container uses";
+    after = [ "docker.service" ];
+    requires = [ "docker.service" ];
+    path = [ config.virtualisation.docker.package ];
+    serviceConfig.Type = "oneshot";
+    script = "docker image prune -af";
+    startAt = "Mon 03:00";
+  };
+}

nixos/modules/restic-alert.py

@@ -0,0 +1,20 @@
+# Mail the tail of a failed unit's journal.
+# Usage: restic-alert <unit> <env file>
+import os, smtplib, socket, subprocess, sys
+from email.message import EmailMessage
+
+unit = sys.argv[1]
+env = dict(l.rstrip("\n").split("=", 1) for l in open(sys.argv[2]) if "=" in l)
+log = subprocess.run(["journalctl", "-u", unit, "-n", "60", "--no-pager", "-o", "cat"],
+                     capture_output=True, text=True).stdout
+
+msg = EmailMessage()
+msg["Subject"] = f"[{socket.gethostname()}] {unit} failed"
+msg["From"] = env["ALERT_SMTP_USER"]
+msg["To"] = env["ALERT_RECIPIENT"]
+msg.set_content(log)
+
+with smtplib.SMTP(env["ALERT_SMTP_HOST"], int(env["ALERT_SMTP_PORT"]), timeout=30) as s:
+    s.starttls()
+    s.login(env["ALERT_SMTP_USER"], env["ALERT_SMTP_PASSWORD"])
+    s.send_message(msg)

renovate.json

@@ -1,16 +1,15 @@
 {
   "$schema": "https://docs.renovatebot.com/renovate-schema.json",
-  "extends": ["config:recommended", ":disableDependencyDashboard"],
+  "extends": [
-  "timezone": "Europe/Helsinki",
+    "local>asxpi/renovate"
-  "labels": ["dependencies"],
+  ],
-  "prHourlyLimit": 0,
-  "prConcurrentLimit": 10,
-  "commitMessagePrefix": "deps:",
   "customManagers": [
     {
       "description": "Versions in Ansible vars, marked with a comment: # renovate: datasource=<ds> depName=<name>",
       "customType": "regex",
-      "managerFilePatterns": ["/^ansible/.+\\.ya?ml$/"],
+      "managerFilePatterns": [
+        "/^ansible/.+\\.ya?ml$/"
+      ],
       "matchStrings": [
         "# renovate: datasource=(?<datasource>\\S+) depName=(?<depName>\\S+)\\s+[a-z_0-9]+:\\s*\"?(?<currentValue>[^\"\\s]+)\"?"
       ]
@@ -19,23 +18,28 @@
   "packageRules": [
     {
       "description": "Paper needs Java 21; the tag is a floating JRE line",
-      "matchPackageNames": ["eclipse-temurin"],
+      "matchPackageNames": [
+        "eclipse-temurin"
+      ],
       "enabled": false
     },
     {
       "description": "Every job runs this image and a nix-store volume tied to its version",
-      "matchPackageNames": ["nixos/nix"],
+      "matchPackageNames": [
-      "prBodyNotes": ["After merging, remove the old `nix-store-<old version>` volume on asxpio-services (see forgejo_runner.yaml)."]
+        "nixos/nix"
+      ],
+      "prBodyNotes": [
+        "After merging, remove the old `nix-store-<old version>` volume on asxpio-services (see forgejo_runner.yaml)."
+      ]
     },
     {
       "description": "A runner change recreates the runner mid-job",
-      "matchPackageNames": ["code.forgejo.org/forgejo/runner"],
+      "matchPackageNames": [
-      "prBodyNotes": ["The deploy job that applies this shows as cancelled; the next run confirms the result."]
+        "code.forgejo.org/forgejo/runner"
-    },
+      ],
-    {
+      "prBodyNotes": [
-      "description": "Renovate itself releases daily; once a month is enough",
+        "The deploy job that applies this shows as cancelled; the next run confirms the result."
-      "matchPackageNames": ["ghcr.io/renovatebot/renovate"],
+      ]
-      "schedule": ["* * 1 * *"]
     }
   ]
 }