ci: run Renovate from asxpi/renovate

Renovate now runs from its own repository for every repo, with the shared rules in its preset. Extend that preset here and keep only what is specific to this repo (the Ansible var manager, the Paper JRE, the runner and nix notes); drop the workflow that ran it for iac alone. Assisted-by: Claude:opus-5-5
2026-11-13 00:00:06 +04:00 · 2026-11-13 00:00:06 +04:00 · 71047d179d
commit 71047d179d
parent 131124e0de
15 changed files with 705 additions and 284 deletions
--- a/.forgejo/workflows/renovate.yaml
+++ b/.forgejo/workflows/renovate.yaml
@ -1,32 +0,0 @@
-name: renovate
-
-# Opens a PR per available version bump (image tags, restic, ...), with
-# the upstream release notes in the PR body. Configuration: renovate.json.
-#
-# Secrets: RENOVATE_TOKEN (Forgejo token of the renovate-bot user, a
-# collaborator with write access) and GH_COM_TOKEN (read-only github.com
-# token, only for fetching release notes; without it PRs have none).
-
-on:
-  schedule:
-    - cron: "17 5 * * *"
-  workflow_dispatch:
-
-jobs:
-  renovate:
-    runs-on: ubuntu-latest
-    container:
-      image: ghcr.io/renovatebot/renovate:44.236.2
-    steps:
-      - name: Run Renovate
-        env:
-          RENOVATE_PLATFORM: forgejo
-          RENOVATE_ENDPOINT: https://git.asxp.io/api/v1
-          RENOVATE_REPOSITORIES: asxpi/iac
-          RENOVATE_TOKEN: ${{ secrets.RENOVATE_TOKEN }}
-          RENOVATE_GITHUB_COM_TOKEN: ${{ secrets.GH_COM_TOKEN }}
-          RENOVATE_GIT_AUTHOR: "Renovate Bot <renovate-bot@noreply.git.asxp.io>"
-          RENOVATE_ONBOARDING: "false"
-          RENOVATE_REQUIRE_CONFIG: required
-          LOG_LEVEL: info
-        run: renovate
--- a/nixos/flake.nix
+++ b/nixos/flake.nix
@ -30,6 +30,8 @@
            sops-nix.nixosModules.sops
            ./modules/base.nix
            ./modules/hetzner.nix
+            ./modules/docker.nix
+            ./modules/backup.nix
          ]
          ++ modules;
        };
--- a/nixos/hosts/asxpio-services/containers.nix
+++ b/nixos/hosts/asxpio-services/containers.nix
@ -1,183 +1,81 @@
-# Containers run under Podman, one systemd unit each
-# (podman-<name>.service). Game data stays in /opt and is not managed here.
-{
-  config,
-  lib,
-  pkgs,
-  ...
-}:
+# Minecraft and SillyTavern. Their data stays in /opt and is not managed
+# here.
+{ config, ... }:
 let
-  podman = lib.getExe pkgs.podman;
-  # Time for a Minecraft server to save the world before it is killed
+  # Paper needs Java 21; the tag is a floating JRE line
+  javaImage = "docker.io/library/eclipse-temurin:21-jre";
+  paperJar = "paper-1.21.11-54.jar";
+  # Time for the server to save the world before it is killed
  stopTimeout = 120;

-  # Nothing that owns data starts on a freshly installed disk: a game
-  # server would generate a new world where the restore is about to land.
-  # After restoring /opt, `touch /opt/.restored` and start the units.
-  needsData.unitConfig.ConditionPathExists = "/opt/.restored";
-
-  gameServer = needsData // {
-    serviceConfig = {
-      TimeoutStopSec = lib.mkForce (stopTimeout + 30);
-      # A JVM stopped by SIGTERM exits 143 after saving; that is a clean stop
-      SuccessExitStatus = [ 143 ];
-    };
-  };
+  # renovate: datasource=docker depName=ghcr.io/sillytavern/sillytavern
+  sillytavernVersion = "1.19.0";
 in
 {
-  virtualisation.podman = {
-    enable = true;
-    dockerCompat = true; # `docker` runs podman
-    dockerSocket.enable = true; # /run/docker.sock, used by the CI runner
-    defaultNetwork.settings.dns_enabled = true;
-    autoPrune = {
-      enable = true;
-      dates = "Mon 03:00";
-      flags = [ "--all" ];
-    };
-  };
-  virtualisation.containers.registries.search = [ "docker.io" ];
-  virtualisation.oci-containers.backend = "podman";
-
-  sops.secrets."minecraft_fossils/env" = { };
-
-  virtualisation.oci-containers.containers = {
-    # Paper server; the jar and the world live in /opt/minecraft
-    minecraft = {
-      image = "docker.io/library/eclipse-temurin:21-jre";
-      workdir = "/data";
-      cmd = [
-        "java"
-        "-Xmx4G"
-        "-Xms2G"
-        "-jar"
-        "paper-1.21.11-54.jar"
-        "nogui"
-      ];
-      ports = [
-        "25565:25565"
-        "19132:19132/udp"
-      ];
-      volumes = [ "/opt/minecraft:/data" ];
-      extraOptions = [
-        "--tty"
-        "--interactive"
-        "--stop-timeout=${toString stopTimeout}"
-      ];
-    };
-
-    minecraft-fossils = {
-      image = "docker.io/itzg/minecraft-server:java21";
-      environment = {
-        EULA = "TRUE";
-        TYPE = "FABRIC";
-        VERSION = "1.20.1";
-        FABRIC_LOADER_VERSION = "0.18.4";
-        MEMORY = "8G";
-        ONLINE_MODE = "FALSE";
-        ENFORCE_WHITELIST = "TRUE";
-        ENFORCE_SECURE_PROFILE = "FALSE";
-        WHITELIST_FILE = "/data/whitelist.json";
-        OPS_FILE = "/data/ops.json";
-        ENABLE_RCON = "TRUE";
-        MOTD = "[mc.czsk.dev] Fossils Archaeology Revival: Genesis";
-        USE_AIKAR_FLAGS = "TRUE";
-      };
-      environmentFiles = [ config.sops.secrets."minecraft_fossils/env".path ];
-      ports = [ "25566:25565" ];
-      volumes = [ "/opt/minecraft-fossils/data:/data" ];
-      extraOptions = [
-        "--tty"
-        "--interactive"
-        "--network=fossils"
-        "--stop-timeout=${toString stopTimeout}"
-      ];
-    };
-
-    # Copies whitelist, ops and ban lists from the Paper server
-    minecraft-fossils-wlsync = {
-      image = "docker.io/library/alpine:3";
-      dependsOn = [ "minecraft-fossils" ];
-      environment = {
-        RCON_HOST = "minecraft-fossils";
-        INTERVAL = "15";
-      };
-      environmentFiles = [ config.sops.secrets."minecraft_fossils/env".path ];
-      volumes = [
-        "/opt/minecraft:/src:ro"
-        "/opt/minecraft-fossils/data:/dst"
-        "${./wlsync.sh}:/wlsync.sh:ro"
-      ];
-      cmd = [
-        "sh"
-        "/wlsync.sh"
-      ];
-      extraOptions = [ "--network=fossils" ];
-    };
-
-    # Reachable on the WireGuard address only
-    sillytavern = {
-      image = "ghcr.io/sillytavern/sillytavern:1.19.0";
-      ports = [ "10.20.0.20:8000:8000" ];
-      volumes = [
-        "/opt/sillytavern/config:/home/node/app/config"
-        "/opt/sillytavern/data:/home/node/app/data"
-        "/opt/sillytavern/plugins:/home/node/app/plugins"
-      ];
+  # Paper server; the jar and the world live in /opt/minecraft. Not an
+  # oci-containers unit: the console needs a TTY and an open stdin, and
+  # `docker run -it` refuses to start without a terminal. The container
+  # is created first and then attached to, output only.
+  #
+  #   docker attach minecraft        console; leave with Ctrl-p Ctrl-q
+  systemd.services.docker-minecraft = {
+    description = "Minecraft (Paper)";
+    wantedBy = [ "multi-user.target" ];
+    wants = [ "network-online.target" ];
+    # requires: Docker restarting stops the server first, with time to save
+    requires = [ "docker.service" ];
+    after = [
+      "docker.service"
+      "network-online.target"
+    ];
+    unitConfig.ConditionPathExists = "/opt/.restored";
+    path = [ config.virtualisation.docker.package ];
+    preStart = ''
+      docker rm -f minecraft || true
+      docker create --name minecraft --tty --interactive \
+        --stop-timeout ${toString stopTimeout} \
+        --workdir /data --volume /opt/minecraft:/data \
+        --publish 25565:25565 --publish 19132:19132/udp \
+        ${javaImage} java -Xmx4G -Xms2G -jar ${paperJar} nogui
+    '';
+    script = "exec docker start --attach minecraft";
+    preStop = "docker stop minecraft || true";
+    postStop = "docker rm -f minecraft || true";
+    serviceConfig = {
+      TimeoutStartSec = 0; # the first start pulls the image
+      TimeoutStopSec = stopTimeout + 30;
+      # A JVM stopped by SIGTERM exits 143 after saving; that is a clean stop
+      SuccessExitStatus = [ 143 ];
+      Restart = "on-failure";
    };
  };

-  systemd.services = {
-    # Network shared by the Fossils server and its sidecar
-    podman-network-fossils = {
-      description = "Podman network fossils";
-      wantedBy = [ "multi-user.target" ];
-      before = [
-        "podman-minecraft-fossils.service"
-        "podman-minecraft-fossils-wlsync.service"
-      ];
-      requiredBy = [
-        "podman-minecraft-fossils.service"
-        "podman-minecraft-fossils-wlsync.service"
-      ];
-      serviceConfig = {
-        Type = "oneshot";
-        RemainAfterExit = true;
-      };
-      script = "${podman} network exists fossils || ${podman} network create fossils";
-    };
-
-    podman-minecraft = gameServer;
-    podman-minecraft-fossils = gameServer;
-    podman-minecraft-fossils-wlsync = needsData;
-
-    # The WireGuard address must exist before a port can be bound to it
-    podman-sillytavern = needsData // {
-      after = [ "wireguard-asxpio-services.service" ];
-      requires = [ "wireguard-asxpio-services.service" ];
-    };
-
-    minecraft-fossils-restart = {
-      description = "Daily restart of minecraft-fossils";
-      serviceConfig.Type = "oneshot";
-      script = "systemctl restart podman-minecraft-fossils.service";
-    };
+  # Reachable on the WireGuard address only
+  virtualisation.oci-containers.containers.sillytavern = {
+    image = "ghcr.io/sillytavern/sillytavern:${sillytavernVersion}";
+    ports = [ "10.20.0.20:8000:8000" ];
+    volumes = [
+      "/opt/sillytavern/config:/home/node/app/config"
+      "/opt/sillytavern/data:/home/node/app/data"
+      "/opt/sillytavern/plugins:/home/node/app/plugins"
+    ];
  };

-  systemd.timers.minecraft-fossils-restart = {
-    description = "Restart minecraft-fossils daily at 08:00 Europe/Tallinn";
-    wantedBy = [ "timers.target" ];
-    timerConfig = {
-      OnCalendar = "*-*-* 08:00:00 Europe/Tallinn";
-      Persistent = true;
-    };
+  # The WireGuard address must exist before a port can be bound to it
+  systemd.services.docker-sillytavern = {
+    unitConfig.ConditionPathExists = "/opt/.restored";
+    after = [ "wireguard-asxpio-services.service" ];
+    requires = [ "wireguard-asxpio-services.service" ];
  };

+  # Published ports reach the containers without passing these rules over
+  # IPv4. Over IPv6 Docker's proxy listens on the host, so they are needed.
+  networking.firewall.allowedTCPPorts = [ 25565 ];
+  networking.firewall.allowedUDPPorts = [ 19132 ];
+
  systemd.tmpfiles.rules = [
    "d /opt 0755 root root -"
    "d /opt/minecraft 0755 root root -"
-    "d /opt/minecraft-fossils 0755 asxpi users -"
-    "d /opt/minecraft-fossils/data 0755 asxpi users -"
    "d /opt/sillytavern/config 0755 root root -"
    "d /opt/sillytavern/data 0755 root root -"
    "d /opt/sillytavern/plugins 0755 root root -"
--- a/nixos/hosts/asxpio-services/default.nix
+++ b/nixos/hosts/asxpio-services/default.nix
@ -1,9 +1,12 @@
-# asxpio-services: CI runner, Minecraft, SillyTavern.
+# asxpio-services: CI runner, Minecraft, XMPP, Xray, SillyTavern.
 { config, lib, ... }:
 {
  imports = [
    ./containers.nix
+    ./replica.nix
    ./runner.nix
+    ./xmpp.nix
+    ./xray.nix
  ];

  networking.hostName = lib.mkDefault "asxpio-services";
@ -59,27 +62,26 @@
  };

  # --- Firewall -----------------------------------------------------------
-  # The Hetzner firewall is the outer layer. Published container ports are
-  # forwarded by Podman and do not pass through these input rules.
-  networking.firewall = {
-    allowedTCPPorts = [
-      25565
-      25566
+  # The Hetzner firewall is the outer layer. Each service opens its own
+  # ports next to its definition.
+  networking.firewall.interfaces = {
+    asxpio-services.allowedTCPPorts = [
+      22
+      8000
+    ];
+    enp7s0.allowedTCPPorts = [ 22 ];
+  };
+
+  # --- Backup -------------------------------------------------------------
+  # The Paper world is copied live (RCON is off, so there is no save-off);
+  # its autosave keeps that close enough. What Paper downloads again is
+  # left out.
+  asxpio.backup = {
+    schedule = "*-*-* 04:15:00";
+    excludes = [
+      "/opt/minecraft/cache"
+      "/opt/minecraft/libraries"
+      "/opt/minecraft/versions"
    ];
-    allowedUDPPorts = [ 19132 ];
-    interfaces = {
-      asxpio-services.allowedTCPPorts = [
-        22
-        8000
-      ];
-      enp7s0.allowedTCPPorts = [ 22 ];
-      # CI job containers reach the runner's cache server on the host
-      "podman+".allowedTCPPortRanges = [
-        {
-          from = 1024;
-          to = 65535;
-        }
-      ];
-    };
  };
 }
--- a/nixos/hosts/asxpio-services/replica.nix
+++ b/nixos/hosts/asxpio-services/replica.nix
@ -0,0 +1,93 @@
+# Daily one-way replica of the hel1 buckets into their fsn1 twins.
+# restic/ is excluded: restic writes its own, independent fsn1 repository.
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  pairs = [
+    {
+      src = "asxpio-backups-main";
+      dst = "asxpio-backups-repl";
+      exclude = [ "restic/**" ];
+    }
+    {
+      src = "asxpio-terraform-main";
+      dst = "asxpio-terraform-repl";
+    }
+    {
+      src = "k8s-asxpio-main";
+      dst = "k8s-asxpio-repl";
+    }
+  ];
+
+  # A run that would delete more than this stops and fails (and mails):
+  # a mistake on the hel1 side must not empty the replica the next morning
+  maxDelete = 1000;
+
+  secret = name: config.sops.placeholder.${name};
+  remote = name: region: extra: ''
+    RCLONE_CONFIG_${name}_TYPE=s3
+    RCLONE_CONFIG_${name}_PROVIDER=Other
+    RCLONE_CONFIG_${name}_ENDPOINT=https://${region}.your-objectstorage.com
+    RCLONE_CONFIG_${name}_REGION=${region}
+    RCLONE_CONFIG_${name}_ACCESS_KEY_ID=${secret "s3/access_key"}
+    RCLONE_CONFIG_${name}_SECRET_ACCESS_KEY=${secret "s3/secret_key"}
+    ${extra}'';
+
+  replicate = pkgs.writeShellApplication {
+    name = "s3-replicate";
+    runtimeInputs = [ pkgs.rclone ];
+    # No errexit: a failing pair does not stop the others, but fails the run
+    bashOptions = [
+      "nounset"
+      "pipefail"
+    ];
+    text = ''
+      failed=0
+    ''
+    + lib.concatMapStrings (p: ''
+      echo "s3-replicate: ${p.src} -> ${p.dst}"
+      rclone sync hel1:${p.src} fsn1:${p.dst} \
+        --fast-list --checkers 16 --transfers 8 --max-delete ${toString maxDelete} \
+        ${lib.concatMapStringsSep " " (e: "--exclude ${lib.escapeShellArg e}") (p.exclude or [ ])} \
+        --stats-one-line -v || failed=1
+    '') pairs
+    + ''
+      exit $failed
+    '';
+  };
+in
+{
+  # The object storage remotes hel1: and fsn1:, as rclone environment
+  # variables. The migration's restore script reads them too.
+  sops.templates."s3-replicate-env".content =
+    remote "HEL1" "hel1" ""
+    + remote "FSN1" "fsn1" ''
+      RCLONE_CONFIG_FSN1_NO_CHECK_BUCKET=true
+    '';
+
+  systemd.services.s3-replicate = {
+    description = "Replicate the hel1 buckets to fsn1";
+    wants = [ "network-online.target" ];
+    after = [ "network-online.target" ];
+    onFailure = [ "restic-alert@%n.service" ];
+    serviceConfig = {
+      Type = "oneshot";
+      ExecStart = lib.getExe replicate;
+      EnvironmentFile = config.sops.templates."s3-replicate-env".path;
+      TimeoutStartSec = "12h";
+    };
+  };
+  systemd.timers.s3-replicate = {
+    description = "Daily hel1 -> fsn1 bucket replica";
+    wantedBy = [ "timers.target" ];
+    timerConfig = {
+      OnCalendar = "*-*-* 06:00:00";
+      RandomizedDelaySec = "15m";
+      Persistent = true;
+    };
+  };
+}
--- a/nixos/hosts/asxpio-services/runner.nix
+++ b/nixos/hosts/asxpio-services/runner.nix
@ -1,13 +1,22 @@
 # Forgejo Actions runner. It runs as a container and starts job containers
-# through Podman's Docker-compatible socket. Its registration lives in
+# through the Docker socket. Its registration lives in
 # /opt/forgejo-runner/data/.runner and is state, not configuration.
+#
+# Deploying a change to it restarts the runner under the job that deploys:
+# the change lands, that job shows as cancelled, and the next run confirms
+# the result.
 { pkgs, ... }:
 let
+  # renovate: datasource=docker depName=code.forgejo.org/forgejo/runner
+  runnerVersion = "13";
+
  # Image behind the nix-latest label. Jobs mount a nix-store volume over
  # /nix, seeded from this image on first use; the image's /bin/sh and
  # /etc/nix/nix.conf link into that store, so a volume from another
-  # version breaks them. Bumping this also switches to a fresh volume.
-  nixVersion = "2.35.3";
+  # version breaks them. Bumping this also switches to a fresh volume;
+  # remove the old one afterwards: docker volume rm nix-store-<old version>
+  # renovate: datasource=docker depName=nixos/nix
+  nixVersion = "2.35.6";

  settings = {
    log = {
@ -65,8 +74,8 @@ let
 in
 {
  virtualisation.oci-containers.containers.forgejo-runner = {
-    image = "code.forgejo.org/forgejo/runner:11";
-    user = "0:0"; # needs the container socket
+    image = "code.forgejo.org/forgejo/runner:${runnerVersion}";
+    user = "0:0"; # needs the Docker socket
    cmd = [
      "forgejo-runner"
      "daemon"
@ -77,17 +86,29 @@ in
    volumes = [
      "/opt/forgejo-runner/data:/data"
      "${configFile}:/config.yaml:ro"
-      "/run/podman/podman.sock:/var/run/docker.sock"
+      "/var/run/docker.sock:/var/run/docker.sock"
    ];
    extraOptions = [ "--network=host" ];
  };

-  systemd.services.podman-forgejo-runner = {
-    after = [ "podman.socket" ];
-    requires = [ "podman.socket" ];
-    # Without its registration the runner only crash-loops
-    unitConfig.ConditionPathExists = "/opt/forgejo-runner/data/.runner";
-  };
+  # Without its registration the runner only crash-loops
+  systemd.services.docker-forgejo-runner.unitConfig.ConditionPathExists =
+    "/opt/forgejo-runner/data/.runner";
+
+  # CI job containers reach the runner's cache server on the host
+  networking.firewall.interfaces =
+    let
+      highPorts.allowedTCPPortRanges = [
+        {
+          from = 1024;
+          to = 65535;
+        }
+      ];
+    in
+    {
+      docker0 = highPorts;
+      "br-+" = highPorts;
+    };

  systemd.tmpfiles.rules = [ "d /opt/forgejo-runner/data 0777 root root -" ];
 }
--- a/nixos/hosts/asxpio-services/wlsync.sh
+++ b/nixos/hosts/asxpio-services/wlsync.sh
@ -1,35 +0,0 @@
-#!/bin/sh
-# Mirrors whitelist, ops and ban lists from the Paper server into the
-# Fossils server and tells it to reload them over RCON.
-apk add --no-cache python3 >/dev/null
-cat > /rcon.py <<'PY'
-import socket, struct, sys, os
-host, port, pw, cmd = os.environ["RCON_HOST"], 25575, os.environ["RCON_PASSWORD"], sys.argv[1]
-def pkt(i, t, b): p = struct.pack("<ii", i, t) + b.encode() + b"\x00\x00"; return struct.pack("<i", len(p)) + p
-def recv(s):
-    ln = struct.unpack("<i", s.recv(4))[0]; d = b""
-    while len(d) < ln: d += s.recv(ln - len(d))
-    return struct.unpack("<ii", d[:8])[0], d[8:-2].decode(errors="replace")
-s = socket.create_connection((host, port), timeout=5)
-s.send(pkt(1, 3, pw))
-rid, _ = recv(s)
-if rid == -1: print("auth failed"); sys.exit(1)
-s.send(pkt(2, 2, cmd))
-_, out = recv(s); print(out)
-PY
-echo "[wlsync] started, interval=${INTERVAL}s"
-while true; do
-  for f in whitelist.json ops.json banned-players.json banned-ips.json; do
-    if [ -f /src/$f ] && ! cmp -s /src/$f /dst/$f 2>/dev/null; then
-      cp /src/$f /dst/$f
-      echo "[wlsync] copied $f"
-      case $f in
-        whitelist.json) CMD="whitelist reload" ;;
-        ops.json)       CMD="reload" ;;
-        banned-players.json|banned-ips.json) CMD="kick @a Ban list updated" ;;
-      esac
-      python3 /rcon.py "$CMD" 2>&1 | sed 's/^/[wlsync] rcon: /' || echo "[wlsync] rcon failed"
-    fi
-  done
-  sleep $INTERVAL
-done
--- a/nixos/hosts/asxpio-services/xmpp.nix
+++ b/nixos/hosts/asxpio-services/xmpp.nix
@ -0,0 +1,134 @@
+# another.im XMPP server for asxp.io: ejabberd, apushd (push), tackygram
+# (Telegram gateway). After compose.yaml and compose.tackygram.yaml of
+# https://dev.another.im/another-im/anotherim-server, CERTS=files.
+#
+# Secrets and keys are host data in /opt/xmpp/state, like a game world,
+# and are covered by the restic backup:
+#   secrets.env          ejabberd secrets
+#   certs/server.pem     TLS certificate (asxp.io, conference., xmpp.)
+#   apushd/              push settings and VAPID key
+#   tackygram/{env,key}  gateway credentials and master key
+# Data: Docker volumes xmpp_database, xmpp_tackygram.
+{ config, pkgs, ... }:
+let
+  # renovate: datasource=docker depName=dev.another.im/another-im/anotherim-server
+  ejabberdVersion = "2026.11.07";
+  # renovate: datasource=docker depName=dev.another.im/another-im/apushd
+  apushdVersion = "0.2";
+  # renovate: datasource=docker depName=dev.another.im/another-im/tackygram
+  tackygramVersion = "0.4.3";
+
+  domain = "asxp.io";
+  publicIp = "46.62.228.216";
+  state = "/opt/xmpp/state";
+
+  tcpPorts = [
+    5222 # clients, STARTTLS
+    5223 # clients, direct TLS
+    5269 # other servers
+    5443 # WebSocket, host-meta
+    8448 # Matrix federation
+    3478 # STUN and TURN
+    7777 # SOCKS5 bytestreams proxy
+  ];
+  turnRelay = {
+    from = 49160;
+    to = 49199;
+  };
+  gatewayCalls = {
+    from = 50000;
+    to = 50099;
+  };
+  range = r: "${toString r.from}-${toString r.to}";
+
+  needsData.unitConfig.ConditionPathExists = "/opt/.restored";
+in
+{
+  virtualisation.oci-containers.containers = {
+    xmpp-ejabberd = {
+      image = "dev.another.im/another-im/anotherim-server:${ejabberdVersion}";
+      environment = {
+        EJABBERD_MACRO_XMPP_DOMAIN = domain;
+        EJABBERD_MACRO_TURN_IP = publicIp;
+        EJABBERD_MACRO_POW_DIFFICULTY = "6";
+        EJABBERD_MACRO_REGISTRATION_TIMEOUT = "600";
+        EJABBERD_MACRO_UPLOAD_URL = "https://hel1.your-objectstorage.com/asxpio-xmpp-public/";
+        EJABBERD_MACRO_ACME_URL = "https://acme-v02.api.letsencrypt.org/directory";
+      };
+      environmentFiles = [ "${state}/secrets.env" ];
+      ports =
+        map (p: "${toString p}:${toString p}") tcpPorts
+        ++ [
+          "3478:3478/udp"
+          "${range turnRelay}:${range turnRelay}/udp"
+          "${range gatewayCalls}:${range gatewayCalls}/udp"
+        ];
+      volumes = [
+        "xmpp_database:/opt/ejabberd/database"
+        "xmpp_logs:/opt/ejabberd/logs"
+        "${state}/certs:/opt/ejabberd/conf/certs:ro"
+      ];
+    };
+
+    # Push App Server and Telegram gateway share ejabberd's network
+    # namespace: they reach its component port on 127.0.0.1 only. They
+    # restart with it, since the namespace is gone when it stops.
+    xmpp-apushd = {
+      image = "dev.another.im/another-im/apushd:${apushdVersion}";
+      dependsOn = [ "xmpp-ejabberd" ];
+      networks = [ "container:xmpp-ejabberd" ];
+      volumes = [ "${state}/apushd:/etc/apushd:ro" ];
+    };
+
+    xmpp-tackygram = {
+      image = "dev.another.im/another-im/tackygram:${tackygramVersion}";
+      dependsOn = [ "xmpp-ejabberd" ];
+      networks = [ "container:xmpp-ejabberd" ];
+      environmentFiles = [ "${state}/tackygram/env" ];
+      environment = {
+        CREDENTIALS_DIRECTORY = "/run/credentials";
+        TACKYGRAM_CALL_IP = publicIp;
+        TACKYGRAM_CALL_PORTS = range gatewayCalls;
+      };
+      volumes = [
+        "xmpp_tackygram:/var/lib/tackygram"
+        "${state}/tackygram/key:/run/credentials/gateway-key:ro"
+      ];
+    };
+  };
+
+  systemd.services = {
+    docker-xmpp-ejabberd = needsData;
+    docker-xmpp-apushd = needsData;
+    docker-xmpp-tackygram = needsData;
+  };
+
+  # For IPv6; see containers.nix
+  networking.firewall = {
+    allowedTCPPorts = tcpPorts;
+    allowedUDPPorts = [ 3478 ];
+    allowedUDPPortRanges = [
+      turnRelay
+      gatewayCalls
+    ];
+  };
+
+  systemd.tmpfiles.rules = [ "d ${state} 0700 root root -" ];
+
+  # ejabberd's Mnesia through its own backup command, its SQLite through
+  # the online backup API
+  asxpio.backup = {
+    paths = [
+      "/var/lib/docker/volumes/xmpp_database"
+      "/var/lib/docker/volumes/xmpp_tackygram"
+    ];
+    hookPackages = [
+      config.virtualisation.docker.package
+      pkgs.sqlite
+    ];
+    preHooks = [
+      "docker exec xmpp-ejabberd ejabberdctl backup /opt/ejabberd/database/nightly.backup"
+      ''sqlite3 /var/lib/docker/volumes/xmpp_database/_data/ejabberd.sqlite ".backup '$STAGE/ejabberd.sqlite'"''
+    ];
+  };
+}
--- a/nixos/hosts/asxpio-services/xray.nix
+++ b/nixos/hosts/asxpio-services/xray.nix
@ -0,0 +1,32 @@
+# Xray VLESS Reality proxy on :443, host network. Root in the container
+# only to read the 0400 config and bind :443; every other capability is
+# dropped.
+{ config, ... }:
+let
+  # renovate: datasource=docker depName=ghcr.io/xtls/xray-core
+  xrayVersion = "26.11.9";
+in
+{
+  # The whole config.json; a new one restarts the container
+  sops.secrets."xray/config".restartUnits = [ "docker-xray.service" ];
+
+  virtualisation.oci-containers.containers.xray = {
+    image = "ghcr.io/xtls/xray-core:${xrayVersion}";
+    cmd = [
+      "run"
+      "-c"
+      "/etc/xray/config.json"
+    ];
+    user = "0";
+    volumes = [ "${config.sops.secrets."xray/config".path}:/etc/xray/config.json:ro" ];
+    extraOptions = [
+      "--network=host"
+      "--cap-drop=ALL"
+      "--cap-add=NET_BIND_SERVICE"
+      "--security-opt=no-new-privileges:true"
+      "--read-only"
+    ];
+  };
+
+  networking.firewall.allowedTCPPorts = [ 443 ];
+}
--- a/nixos/hosts/rehearsal.nix
+++ b/nixos/hosts/rehearsal.nix
@ -1,6 +1,9 @@
 # Overrides that turn asxpio-services into a throwaway test machine:
 # plain DHCP, SSH from anywhere, no WireGuard, dummy secrets.
 { lib, ... }:
+let
+  manual.autoStart = lib.mkForce false;
+in
 {
  networking.hostName = "rehearsal";
  sops.defaultSopsFile = ../secrets/rehearsal.yaml;
@ -13,11 +16,26 @@
  };

  networking.wireguard.interfaces = lib.mkForce { };
-  systemd.services.podman-sillytavern = {
+  systemd.services.docker-sillytavern = {
    after = lib.mkForce [ ];
    requires = lib.mkForce [ ];
  };
-  virtualisation.oci-containers.containers.sillytavern.ports = lib.mkForce [ "127.0.0.1:8000:8000" ];
+  virtualisation.oci-containers.containers = {
+    sillytavern.ports = lib.mkForce [ "127.0.0.1:8000:8000" ];
+
+    # With production data restored these would act as production: a
+    # second asxp.io XMPP server and Telegram session, a runner taking
+    # real CI jobs. Start them by hand, and only with test data.
+    xmpp-ejabberd = manual;
+    xmpp-apushd = manual;
+    xmpp-tackygram = manual;
+    forgejo-runner = manual;
+    xray = manual; # its config is a dummy
+  };
+
+  # No backups of, or from, a test machine
+  systemd.timers.restic-backup.enable = false;
+  systemd.timers.s3-replicate.enable = false;

  services.openssh.openFirewall = lib.mkForce true;
  services.openssh.settings.AllowUsers = [ "root" ];
--- a/nixos/modules/backup.nix
+++ b/nixos/modules/backup.nix
@ -0,0 +1,239 @@
+# Nightly restic backup to Hetzner Object Storage, one repository per
+# region, written independently: losing one region or one repository
+# leaves the other complete. The same script and command as the Ansible
+# role restic_backup installs on asxpio-ops:
+#
+#   restic-backup hel1 snapshots
+#   restic-backup fsn1 restore latest --target /tmp/r --include /opt/xmpp
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.asxpio.backup;
+  host = config.networking.hostName;
+
+  # The release the Ansible role pins (restic_version), so both hosts
+  # write their repositories with the same version. nixpkgs 26.05 has an
+  # older one. To bump: change the version, then take the new hash from
+  #   nix store prefetch-file <url>
+  resticVersion = "0.19.1";
+  restic =
+    pkgs.runCommand "restic-${resticVersion}"
+      {
+        src = pkgs.fetchurl {
+          url = "https://github.com/restic/restic/releases/download/v${resticVersion}/restic_${resticVersion}_linux_amd64.bz2";
+          hash = "sha256-9BVBViTcxFLyoCuMM2QXkajG1tO2W7s1Q/z5olFRWFw=";
+        };
+        nativeBuildInputs = [ pkgs.bzip2 ];
+      }
+      ''
+        mkdir -p $out/bin
+        bunzip2 -c $src > $out/bin/restic
+        chmod 755 $out/bin/restic
+      '';
+
+  repositories = {
+    hel1 = "s3:https://hel1.your-objectstorage.com/asxpio-backups-main/restic/${host}";
+    fsn1 = "s3:https://fsn1.your-objectstorage.com/asxpio-backups-repl/restic/${host}";
+  };
+
+  envFile = config.sops.templates."restic-env".path;
+  secret = name: config.sops.placeholder.${name};
+
+  hooks = lib.imap1 (i: hook: ''
+    log "hook ${toString i}/${toString (lib.length cfg.preHooks)}"
+    ( set -e; ${hook} ) || { log "hook failed"; failed=1; }
+  '') cfg.preHooks;
+
+  backup = pkgs.writeShellApplication {
+    name = "restic-backup";
+    runtimeInputs = [
+      restic
+      pkgs.coreutils
+    ]
+    ++ cfg.hookPackages;
+    # No errexit: a failing hook or repository must not stop the others
+    bashOptions = [
+      "nounset"
+      "pipefail"
+    ];
+    text = ''
+      usage() {
+        cat <<'USAGE'
+      restic-backup run                 hooks, then backup + forget to every repository
+      restic-backup init                create the repositories that do not exist yet
+      restic-backup <repo> <args...>    plain restic against one repository, e.g.
+                                        restic-backup hel1 snapshots
+                                        restic-backup fsn1 restore latest --target /tmp/r
+
+      Repositories: ${lib.concatStringsSep ", " (lib.attrNames repositories)}
+      USAGE
+      }
+
+      set -a
+      # shellcheck source=/dev/null
+      . ${envFile}
+      set +a
+      export RESTIC_CACHE_DIR=/var/cache/restic
+      STAGE=${cfg.stageDir}
+
+      declare -A REPOS=(
+      ${lib.concatStrings (lib.mapAttrsToList (name: url: "  [${name}]=\"${url}\"\n") repositories)})
+
+      log() { echo "restic-backup: $*"; }
+
+      case "''${1:-}" in
+      init)
+        for name in "''${!REPOS[@]}"; do
+          if RESTIC_REPOSITORY="''${REPOS[$name]}" restic cat config >/dev/null 2>&1; then
+            log "$name: exists"
+          else
+            RESTIC_REPOSITORY="''${REPOS[$name]}" restic init || exit 1
+            log "$name: created"
+          fi
+        done
+        ;;
+
+      run)
+        failed=0
+        rm -rf "$STAGE"
+        install -d -m 700 "$STAGE"
+        export STAGE
+      ${lib.concatStrings hooks}
+        for name in "''${!REPOS[@]}"; do
+          export RESTIC_REPOSITORY="''${REPOS[$name]}"
+          log "$name: backup"
+          restic backup --host ${host} --one-file-system --exclude-caches \
+            ${lib.concatMapStringsSep " " (e: "--exclude ${lib.escapeShellArg e}") cfg.excludes} \
+            ${lib.escapeShellArgs cfg.paths} "$STAGE" || { failed=1; continue; }
+
+          # Prune rewrites pack files, so costs S3 requests: once a week only
+          prune=()
+          [ "$(date +%a)" = Sun ] && prune=(--prune)
+          log "$name: forget ''${prune[*]}"
+          restic forget --host ${host} --keep-daily 7 --keep-weekly 4 --keep-monthly 6 \
+            "''${prune[@]}" || failed=1
+        done
+
+        rm -rf "$STAGE"
+        exit $failed
+        ;;
+
+      *)
+        [ -n "''${1:-}" ] && [ -n "''${REPOS[$1]:-}" ] || { usage; exit 2; }
+        export RESTIC_REPOSITORY="''${REPOS[$1]}"
+        shift
+        exec restic "$@"
+        ;;
+      esac
+    '';
+  };
+
+  alert = pkgs.writers.writePython3Bin "restic-alert" { doCheck = false; } (
+    builtins.readFile ./restic-alert.py
+  );
+in
+{
+  options.asxpio.backup = {
+    paths = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [
+        "/etc"
+        "/opt"
+      ];
+      description = "What to back up, besides the stage directory.";
+    };
+    excludes = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+    };
+    preHooks = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = ''
+        Shell lines run in order before the backup, with $STAGE set. A
+        failing hook marks the run as failed but the backup still runs.
+      '';
+    };
+    hookPackages = lib.mkOption {
+      type = lib.types.listOf lib.types.package;
+      default = [ ];
+      description = "Programs the hooks call.";
+    };
+    stageDir = lib.mkOption {
+      type = lib.types.str;
+      default = "/var/backups/restic-stage";
+      description = "Dumps written by the hooks land here and are backed up too.";
+    };
+    schedule = lib.mkOption {
+      type = lib.types.str;
+      default = "*-*-* 03:30:00";
+    };
+  };
+
+  config = {
+    sops.secrets = {
+      "restic/password" = { };
+      "s3/access_key" = { };
+      "s3/secret_key" = { };
+      "smtp/user" = { };
+      "smtp/password" = { };
+      "alert/recipient" = { };
+    };
+    sops.templates."restic-env".content = ''
+      RESTIC_PASSWORD=${secret "restic/password"}
+      AWS_ACCESS_KEY_ID=${secret "s3/access_key"}
+      AWS_SECRET_ACCESS_KEY=${secret "s3/secret_key"}
+      ALERT_SMTP_HOST=smtp.fastmail.com
+      ALERT_SMTP_PORT=587
+      ALERT_SMTP_USER=${secret "smtp/user"}
+      ALERT_SMTP_PASSWORD=${secret "smtp/password"}
+      ALERT_RECIPIENT=${secret "alert/recipient"}
+    '';
+
+    environment.systemPackages = [ backup ];
+    systemd.tmpfiles.rules = [
+      "d /var/cache/restic 0700 root root -"
+      "d /var/backups 0755 root root -"
+    ];
+
+    systemd.services.restic-backup = {
+      description = "restic backup to Hetzner Object Storage (${lib.concatStringsSep ", " (lib.attrNames repositories)})";
+      wants = [ "network-online.target" ];
+      after = [
+        "network-online.target"
+        "docker.service"
+      ];
+      onFailure = [ "restic-alert@%n.service" ];
+      serviceConfig = {
+        Type = "oneshot";
+        ExecStart = "${lib.getExe backup} run";
+        Nice = 10;
+        IOSchedulingClass = "idle";
+        TimeoutStartSec = "6h";
+      };
+    };
+    systemd.timers.restic-backup = {
+      description = "Nightly restic backup";
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnCalendar = cfg.schedule;
+        RandomizedDelaySec = "10m";
+        Persistent = true;
+      };
+    };
+
+    # Mails the tail of a failed unit's journal, through Forgejo's SMTP
+    # account. Any unit can use it: OnFailure=restic-alert@%n.service
+    systemd.services."restic-alert@" = {
+      description = "Mail the failure of %i";
+      serviceConfig = {
+        Type = "oneshot";
+        ExecStart = "${lib.getExe alert} %i ${envFile}";
+      };
+    };
+  };
+}
--- a/nixos/modules/base.nix
+++ b/nixos/modules/base.nix
@ -13,7 +13,7 @@ let
    isNormalUser = true;
    extraGroups = [
      "wheel"
-      "podman"
+      "docker"
    ];
    openssh.authorizedKeys.keys = sshKeys;
  };
--- a/nixos/modules/docker.nix
+++ b/nixos/modules/docker.nix
@ -0,0 +1,25 @@
+# Docker, set up as on the Debian hosts. Declared containers become
+# systemd units named docker-<name>.service.
+{ config, ... }:
+{
+  virtualisation.docker = {
+    enable = true;
+    storageDriver = "overlay2";
+    # For containers nobody declared here, such as CI jobs. Declared
+    # containers log to the journal instead.
+    logDriver = "json-file";
+    daemon.settings.log-opts.max-size = "20m";
+  };
+  virtualisation.oci-containers.backend = "docker";
+
+  # Every image update and CI job leaves an image behind
+  systemd.services.docker-image-prune = {
+    description = "Remove images no container uses";
+    after = [ "docker.service" ];
+    requires = [ "docker.service" ];
+    path = [ config.virtualisation.docker.package ];
+    serviceConfig.Type = "oneshot";
+    script = "docker image prune -af";
+    startAt = "Mon 03:00";
+  };
+}
--- a/nixos/modules/restic-alert.py
+++ b/nixos/modules/restic-alert.py
@ -0,0 +1,20 @@
+# Mail the tail of a failed unit's journal.
+# Usage: restic-alert <unit> <env file>
+import os, smtplib, socket, subprocess, sys
+from email.message import EmailMessage
+
+unit = sys.argv[1]
+env = dict(l.rstrip("\n").split("=", 1) for l in open(sys.argv[2]) if "=" in l)
+log = subprocess.run(["journalctl", "-u", unit, "-n", "60", "--no-pager", "-o", "cat"],
+                     capture_output=True, text=True).stdout
+
+msg = EmailMessage()
+msg["Subject"] = f"[{socket.gethostname()}] {unit} failed"
+msg["From"] = env["ALERT_SMTP_USER"]
+msg["To"] = env["ALERT_RECIPIENT"]
+msg.set_content(log)
+
+with smtplib.SMTP(env["ALERT_SMTP_HOST"], int(env["ALERT_SMTP_PORT"]), timeout=30) as s:
+    s.starttls()
+    s.login(env["ALERT_SMTP_USER"], env["ALERT_SMTP_PASSWORD"])
+    s.send_message(msg)
--- a/renovate.json
+++ b/renovate.json
@ -1,16 +1,15 @@
 {
  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
-  "extends": ["config:recommended", ":disableDependencyDashboard"],
-  "timezone": "Europe/Helsinki",
-  "labels": ["dependencies"],
-  "prHourlyLimit": 0,
-  "prConcurrentLimit": 10,
-  "commitMessagePrefix": "deps:",
+  "extends": [
+    "local>asxpi/renovate"
+  ],
  "customManagers": [
    {
      "description": "Versions in Ansible vars, marked with a comment: # renovate: datasource=<ds> depName=<name>",
      "customType": "regex",
-      "managerFilePatterns": ["/^ansible/.+\\.ya?ml$/"],
+      "managerFilePatterns": [
+        "/^ansible/.+\\.ya?ml$/"
+      ],
      "matchStrings": [
        "# renovate: datasource=(?<datasource>\\S+) depName=(?<depName>\\S+)\\s+[a-z_0-9]+:\\s*\"?(?<currentValue>[^\"\\s]+)\"?"
      ]
@ -19,23 +18,28 @@
  "packageRules": [
    {
      "description": "Paper needs Java 21; the tag is a floating JRE line",
-      "matchPackageNames": ["eclipse-temurin"],
+      "matchPackageNames": [
+        "eclipse-temurin"
+      ],
      "enabled": false
    },
    {
      "description": "Every job runs this image and a nix-store volume tied to its version",
-      "matchPackageNames": ["nixos/nix"],
-      "prBodyNotes": ["After merging, remove the old `nix-store-<old version>` volume on asxpio-services (see forgejo_runner.yaml)."]
+      "matchPackageNames": [
+        "nixos/nix"
+      ],
+      "prBodyNotes": [
+        "After merging, remove the old `nix-store-<old version>` volume on asxpio-services (see forgejo_runner.yaml)."
+      ]
    },
    {
      "description": "A runner change recreates the runner mid-job",
-      "matchPackageNames": ["code.forgejo.org/forgejo/runner"],
-      "prBodyNotes": ["The deploy job that applies this shows as cancelled; the next run confirms the result."]
-    },
-    {
-      "description": "Renovate itself releases daily; once a month is enough",
-      "matchPackageNames": ["ghcr.io/renovatebot/renovate"],
-      "schedule": ["* * 1 * *"]
+      "matchPackageNames": [
+        "code.forgejo.org/forgejo/runner"
+      ],
+      "prBodyNotes": [
+        "The deploy job that applies this shows as cancelled; the next run confirms the result."
+      ]
    }
  ]
 }