diff --git a/.forgejo/workflows/renovate.yaml b/.forgejo/workflows/renovate.yaml deleted file mode 100644 index 8120ac5..0000000 --- a/.forgejo/workflows/renovate.yaml +++ /dev/null @@ -1,32 +0,0 @@ -name: renovate - -# Opens a PR per available version bump (image tags, restic, ...), with -# the upstream release notes in the PR body. Configuration: renovate.json. -# -# Secrets: RENOVATE_TOKEN (Forgejo token of the renovate-bot user, a -# collaborator with write access) and GH_COM_TOKEN (read-only github.com -# token, only for fetching release notes; without it PRs have none). - -on: - schedule: - - cron: "17 5 * * *" - workflow_dispatch: - -jobs: - renovate: - runs-on: ubuntu-latest - container: - image: ghcr.io/renovatebot/renovate:44.236.2 - steps: - - name: Run Renovate - env: - RENOVATE_PLATFORM: forgejo - RENOVATE_ENDPOINT: https://git.asxp.io/api/v1 - RENOVATE_REPOSITORIES: asxpi/iac - RENOVATE_TOKEN: ${{ secrets.RENOVATE_TOKEN }} - RENOVATE_GITHUB_COM_TOKEN: ${{ secrets.GH_COM_TOKEN }} - RENOVATE_GIT_AUTHOR: "Renovate Bot " - RENOVATE_ONBOARDING: "false" - RENOVATE_REQUIRE_CONFIG: required - LOG_LEVEL: info - run: renovate diff --git a/nixos/flake.nix b/nixos/flake.nix index 8c90378..51602bc 100644 --- a/nixos/flake.nix +++ b/nixos/flake.nix @@ -30,6 +30,8 @@ sops-nix.nixosModules.sops ./modules/base.nix ./modules/hetzner.nix + ./modules/docker.nix + ./modules/backup.nix ] ++ modules; }; diff --git a/nixos/hosts/asxpio-services/containers.nix b/nixos/hosts/asxpio-services/containers.nix index 010ff52..19c0c0f 100644 --- a/nixos/hosts/asxpio-services/containers.nix +++ b/nixos/hosts/asxpio-services/containers.nix @@ -1,183 +1,81 @@ -# Containers run under Podman, one systemd unit each -# (podman-.service). Game data stays in /opt and is not managed here. -{ - config, - lib, - pkgs, - ... -}: +# Minecraft and SillyTavern. Their data stays in /opt and is not managed +# here. +{ config, ... }: let - podman = lib.getExe pkgs.podman; - # Time for a Minecraft server to save the world before it is killed + # Paper needs Java 21; the tag is a floating JRE line + javaImage = "docker.io/library/eclipse-temurin:21-jre"; + paperJar = "paper-1.21.11-54.jar"; + # Time for the server to save the world before it is killed stopTimeout = 120; - # Nothing that owns data starts on a freshly installed disk: a game - # server would generate a new world where the restore is about to land. - # After restoring /opt, `touch /opt/.restored` and start the units. - needsData.unitConfig.ConditionPathExists = "/opt/.restored"; - - gameServer = needsData // { - serviceConfig = { - TimeoutStopSec = lib.mkForce (stopTimeout + 30); - # A JVM stopped by SIGTERM exits 143 after saving; that is a clean stop - SuccessExitStatus = [ 143 ]; - }; - }; + # renovate: datasource=docker depName=ghcr.io/sillytavern/sillytavern + sillytavernVersion = "1.19.0"; in { - virtualisation.podman = { - enable = true; - dockerCompat = true; # `docker` runs podman - dockerSocket.enable = true; # /run/docker.sock, used by the CI runner - defaultNetwork.settings.dns_enabled = true; - autoPrune = { - enable = true; - dates = "Mon 03:00"; - flags = [ "--all" ]; - }; - }; - virtualisation.containers.registries.search = [ "docker.io" ]; - virtualisation.oci-containers.backend = "podman"; - - sops.secrets."minecraft_fossils/env" = { }; - - virtualisation.oci-containers.containers = { - # Paper server; the jar and the world live in /opt/minecraft - minecraft = { - image = "docker.io/library/eclipse-temurin:21-jre"; - workdir = "/data"; - cmd = [ - "java" - "-Xmx4G" - "-Xms2G" - "-jar" - "paper-1.21.11-54.jar" - "nogui" - ]; - ports = [ - "25565:25565" - "19132:19132/udp" - ]; - volumes = [ "/opt/minecraft:/data" ]; - extraOptions = [ - "--tty" - "--interactive" - "--stop-timeout=${toString stopTimeout}" - ]; - }; - - minecraft-fossils = { - image = "docker.io/itzg/minecraft-server:java21"; - environment = { - EULA = "TRUE"; - TYPE = "FABRIC"; - VERSION = "1.20.1"; - FABRIC_LOADER_VERSION = "0.18.4"; - MEMORY = "8G"; - ONLINE_MODE = "FALSE"; - ENFORCE_WHITELIST = "TRUE"; - ENFORCE_SECURE_PROFILE = "FALSE"; - WHITELIST_FILE = "/data/whitelist.json"; - OPS_FILE = "/data/ops.json"; - ENABLE_RCON = "TRUE"; - MOTD = "[mc.czsk.dev] Fossils Archaeology Revival: Genesis"; - USE_AIKAR_FLAGS = "TRUE"; - }; - environmentFiles = [ config.sops.secrets."minecraft_fossils/env".path ]; - ports = [ "25566:25565" ]; - volumes = [ "/opt/minecraft-fossils/data:/data" ]; - extraOptions = [ - "--tty" - "--interactive" - "--network=fossils" - "--stop-timeout=${toString stopTimeout}" - ]; - }; - - # Copies whitelist, ops and ban lists from the Paper server - minecraft-fossils-wlsync = { - image = "docker.io/library/alpine:3"; - dependsOn = [ "minecraft-fossils" ]; - environment = { - RCON_HOST = "minecraft-fossils"; - INTERVAL = "15"; - }; - environmentFiles = [ config.sops.secrets."minecraft_fossils/env".path ]; - volumes = [ - "/opt/minecraft:/src:ro" - "/opt/minecraft-fossils/data:/dst" - "${./wlsync.sh}:/wlsync.sh:ro" - ]; - cmd = [ - "sh" - "/wlsync.sh" - ]; - extraOptions = [ "--network=fossils" ]; - }; - - # Reachable on the WireGuard address only - sillytavern = { - image = "ghcr.io/sillytavern/sillytavern:1.19.0"; - ports = [ "10.20.0.20:8000:8000" ]; - volumes = [ - "/opt/sillytavern/config:/home/node/app/config" - "/opt/sillytavern/data:/home/node/app/data" - "/opt/sillytavern/plugins:/home/node/app/plugins" - ]; + # Paper server; the jar and the world live in /opt/minecraft. Not an + # oci-containers unit: the console needs a TTY and an open stdin, and + # `docker run -it` refuses to start without a terminal. The container + # is created first and then attached to, output only. + # + # docker attach minecraft console; leave with Ctrl-p Ctrl-q + systemd.services.docker-minecraft = { + description = "Minecraft (Paper)"; + wantedBy = [ "multi-user.target" ]; + wants = [ "network-online.target" ]; + # requires: Docker restarting stops the server first, with time to save + requires = [ "docker.service" ]; + after = [ + "docker.service" + "network-online.target" + ]; + unitConfig.ConditionPathExists = "/opt/.restored"; + path = [ config.virtualisation.docker.package ]; + preStart = '' + docker rm -f minecraft || true + docker create --name minecraft --tty --interactive \ + --stop-timeout ${toString stopTimeout} \ + --workdir /data --volume /opt/minecraft:/data \ + --publish 25565:25565 --publish 19132:19132/udp \ + ${javaImage} java -Xmx4G -Xms2G -jar ${paperJar} nogui + ''; + script = "exec docker start --attach minecraft"; + preStop = "docker stop minecraft || true"; + postStop = "docker rm -f minecraft || true"; + serviceConfig = { + TimeoutStartSec = 0; # the first start pulls the image + TimeoutStopSec = stopTimeout + 30; + # A JVM stopped by SIGTERM exits 143 after saving; that is a clean stop + SuccessExitStatus = [ 143 ]; + Restart = "on-failure"; }; }; - systemd.services = { - # Network shared by the Fossils server and its sidecar - podman-network-fossils = { - description = "Podman network fossils"; - wantedBy = [ "multi-user.target" ]; - before = [ - "podman-minecraft-fossils.service" - "podman-minecraft-fossils-wlsync.service" - ]; - requiredBy = [ - "podman-minecraft-fossils.service" - "podman-minecraft-fossils-wlsync.service" - ]; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - }; - script = "${podman} network exists fossils || ${podman} network create fossils"; - }; - - podman-minecraft = gameServer; - podman-minecraft-fossils = gameServer; - podman-minecraft-fossils-wlsync = needsData; - - # The WireGuard address must exist before a port can be bound to it - podman-sillytavern = needsData // { - after = [ "wireguard-asxpio-services.service" ]; - requires = [ "wireguard-asxpio-services.service" ]; - }; - - minecraft-fossils-restart = { - description = "Daily restart of minecraft-fossils"; - serviceConfig.Type = "oneshot"; - script = "systemctl restart podman-minecraft-fossils.service"; - }; + # Reachable on the WireGuard address only + virtualisation.oci-containers.containers.sillytavern = { + image = "ghcr.io/sillytavern/sillytavern:${sillytavernVersion}"; + ports = [ "10.20.0.20:8000:8000" ]; + volumes = [ + "/opt/sillytavern/config:/home/node/app/config" + "/opt/sillytavern/data:/home/node/app/data" + "/opt/sillytavern/plugins:/home/node/app/plugins" + ]; }; - systemd.timers.minecraft-fossils-restart = { - description = "Restart minecraft-fossils daily at 08:00 Europe/Tallinn"; - wantedBy = [ "timers.target" ]; - timerConfig = { - OnCalendar = "*-*-* 08:00:00 Europe/Tallinn"; - Persistent = true; - }; + # The WireGuard address must exist before a port can be bound to it + systemd.services.docker-sillytavern = { + unitConfig.ConditionPathExists = "/opt/.restored"; + after = [ "wireguard-asxpio-services.service" ]; + requires = [ "wireguard-asxpio-services.service" ]; }; + # Published ports reach the containers without passing these rules over + # IPv4. Over IPv6 Docker's proxy listens on the host, so they are needed. + networking.firewall.allowedTCPPorts = [ 25565 ]; + networking.firewall.allowedUDPPorts = [ 19132 ]; + systemd.tmpfiles.rules = [ "d /opt 0755 root root -" "d /opt/minecraft 0755 root root -" - "d /opt/minecraft-fossils 0755 asxpi users -" - "d /opt/minecraft-fossils/data 0755 asxpi users -" "d /opt/sillytavern/config 0755 root root -" "d /opt/sillytavern/data 0755 root root -" "d /opt/sillytavern/plugins 0755 root root -" diff --git a/nixos/hosts/asxpio-services/default.nix b/nixos/hosts/asxpio-services/default.nix index 452520d..4caab83 100644 --- a/nixos/hosts/asxpio-services/default.nix +++ b/nixos/hosts/asxpio-services/default.nix @@ -1,9 +1,12 @@ -# asxpio-services: CI runner, Minecraft, SillyTavern. +# asxpio-services: CI runner, Minecraft, XMPP, Xray, SillyTavern. { config, lib, ... }: { imports = [ ./containers.nix + ./replica.nix ./runner.nix + ./xmpp.nix + ./xray.nix ]; networking.hostName = lib.mkDefault "asxpio-services"; @@ -59,27 +62,26 @@ }; # --- Firewall ----------------------------------------------------------- - # The Hetzner firewall is the outer layer. Published container ports are - # forwarded by Podman and do not pass through these input rules. - networking.firewall = { - allowedTCPPorts = [ - 25565 - 25566 + # The Hetzner firewall is the outer layer. Each service opens its own + # ports next to its definition. + networking.firewall.interfaces = { + asxpio-services.allowedTCPPorts = [ + 22 + 8000 + ]; + enp7s0.allowedTCPPorts = [ 22 ]; + }; + + # --- Backup ------------------------------------------------------------- + # The Paper world is copied live (RCON is off, so there is no save-off); + # its autosave keeps that close enough. What Paper downloads again is + # left out. + asxpio.backup = { + schedule = "*-*-* 04:15:00"; + excludes = [ + "/opt/minecraft/cache" + "/opt/minecraft/libraries" + "/opt/minecraft/versions" ]; - allowedUDPPorts = [ 19132 ]; - interfaces = { - asxpio-services.allowedTCPPorts = [ - 22 - 8000 - ]; - enp7s0.allowedTCPPorts = [ 22 ]; - # CI job containers reach the runner's cache server on the host - "podman+".allowedTCPPortRanges = [ - { - from = 1024; - to = 65535; - } - ]; - }; }; } diff --git a/nixos/hosts/asxpio-services/replica.nix b/nixos/hosts/asxpio-services/replica.nix new file mode 100644 index 0000000..a7648cf --- /dev/null +++ b/nixos/hosts/asxpio-services/replica.nix @@ -0,0 +1,93 @@ +# Daily one-way replica of the hel1 buckets into their fsn1 twins. +# restic/ is excluded: restic writes its own, independent fsn1 repository. +{ + config, + lib, + pkgs, + ... +}: +let + pairs = [ + { + src = "asxpio-backups-main"; + dst = "asxpio-backups-repl"; + exclude = [ "restic/**" ]; + } + { + src = "asxpio-terraform-main"; + dst = "asxpio-terraform-repl"; + } + { + src = "k8s-asxpio-main"; + dst = "k8s-asxpio-repl"; + } + ]; + + # A run that would delete more than this stops and fails (and mails): + # a mistake on the hel1 side must not empty the replica the next morning + maxDelete = 1000; + + secret = name: config.sops.placeholder.${name}; + remote = name: region: extra: '' + RCLONE_CONFIG_${name}_TYPE=s3 + RCLONE_CONFIG_${name}_PROVIDER=Other + RCLONE_CONFIG_${name}_ENDPOINT=https://${region}.your-objectstorage.com + RCLONE_CONFIG_${name}_REGION=${region} + RCLONE_CONFIG_${name}_ACCESS_KEY_ID=${secret "s3/access_key"} + RCLONE_CONFIG_${name}_SECRET_ACCESS_KEY=${secret "s3/secret_key"} + ${extra}''; + + replicate = pkgs.writeShellApplication { + name = "s3-replicate"; + runtimeInputs = [ pkgs.rclone ]; + # No errexit: a failing pair does not stop the others, but fails the run + bashOptions = [ + "nounset" + "pipefail" + ]; + text = '' + failed=0 + '' + + lib.concatMapStrings (p: '' + echo "s3-replicate: ${p.src} -> ${p.dst}" + rclone sync hel1:${p.src} fsn1:${p.dst} \ + --fast-list --checkers 16 --transfers 8 --max-delete ${toString maxDelete} \ + ${lib.concatMapStringsSep " " (e: "--exclude ${lib.escapeShellArg e}") (p.exclude or [ ])} \ + --stats-one-line -v || failed=1 + '') pairs + + '' + exit $failed + ''; + }; +in +{ + # The object storage remotes hel1: and fsn1:, as rclone environment + # variables. The migration's restore script reads them too. + sops.templates."s3-replicate-env".content = + remote "HEL1" "hel1" "" + + remote "FSN1" "fsn1" '' + RCLONE_CONFIG_FSN1_NO_CHECK_BUCKET=true + ''; + + systemd.services.s3-replicate = { + description = "Replicate the hel1 buckets to fsn1"; + wants = [ "network-online.target" ]; + after = [ "network-online.target" ]; + onFailure = [ "restic-alert@%n.service" ]; + serviceConfig = { + Type = "oneshot"; + ExecStart = lib.getExe replicate; + EnvironmentFile = config.sops.templates."s3-replicate-env".path; + TimeoutStartSec = "12h"; + }; + }; + systemd.timers.s3-replicate = { + description = "Daily hel1 -> fsn1 bucket replica"; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = "*-*-* 06:00:00"; + RandomizedDelaySec = "15m"; + Persistent = true; + }; + }; +} diff --git a/nixos/hosts/asxpio-services/runner.nix b/nixos/hosts/asxpio-services/runner.nix index c69258e..2d1907a 100644 --- a/nixos/hosts/asxpio-services/runner.nix +++ b/nixos/hosts/asxpio-services/runner.nix @@ -1,13 +1,22 @@ # Forgejo Actions runner. It runs as a container and starts job containers -# through Podman's Docker-compatible socket. Its registration lives in +# through the Docker socket. Its registration lives in # /opt/forgejo-runner/data/.runner and is state, not configuration. +# +# Deploying a change to it restarts the runner under the job that deploys: +# the change lands, that job shows as cancelled, and the next run confirms +# the result. { pkgs, ... }: let + # renovate: datasource=docker depName=code.forgejo.org/forgejo/runner + runnerVersion = "13"; + # Image behind the nix-latest label. Jobs mount a nix-store volume over # /nix, seeded from this image on first use; the image's /bin/sh and # /etc/nix/nix.conf link into that store, so a volume from another - # version breaks them. Bumping this also switches to a fresh volume. - nixVersion = "2.35.3"; + # version breaks them. Bumping this also switches to a fresh volume; + # remove the old one afterwards: docker volume rm nix-store- + # renovate: datasource=docker depName=nixos/nix + nixVersion = "2.35.6"; settings = { log = { @@ -65,8 +74,8 @@ let in { virtualisation.oci-containers.containers.forgejo-runner = { - image = "code.forgejo.org/forgejo/runner:11"; - user = "0:0"; # needs the container socket + image = "code.forgejo.org/forgejo/runner:${runnerVersion}"; + user = "0:0"; # needs the Docker socket cmd = [ "forgejo-runner" "daemon" @@ -77,17 +86,29 @@ in volumes = [ "/opt/forgejo-runner/data:/data" "${configFile}:/config.yaml:ro" - "/run/podman/podman.sock:/var/run/docker.sock" + "/var/run/docker.sock:/var/run/docker.sock" ]; extraOptions = [ "--network=host" ]; }; - systemd.services.podman-forgejo-runner = { - after = [ "podman.socket" ]; - requires = [ "podman.socket" ]; - # Without its registration the runner only crash-loops - unitConfig.ConditionPathExists = "/opt/forgejo-runner/data/.runner"; - }; + # Without its registration the runner only crash-loops + systemd.services.docker-forgejo-runner.unitConfig.ConditionPathExists = + "/opt/forgejo-runner/data/.runner"; + + # CI job containers reach the runner's cache server on the host + networking.firewall.interfaces = + let + highPorts.allowedTCPPortRanges = [ + { + from = 1024; + to = 65535; + } + ]; + in + { + docker0 = highPorts; + "br-+" = highPorts; + }; systemd.tmpfiles.rules = [ "d /opt/forgejo-runner/data 0777 root root -" ]; } diff --git a/nixos/hosts/asxpio-services/wlsync.sh b/nixos/hosts/asxpio-services/wlsync.sh deleted file mode 100644 index 6fe4b60..0000000 --- a/nixos/hosts/asxpio-services/wlsync.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/sh -# Mirrors whitelist, ops and ban lists from the Paper server into the -# Fossils server and tells it to reload them over RCON. -apk add --no-cache python3 >/dev/null -cat > /rcon.py <<'PY' -import socket, struct, sys, os -host, port, pw, cmd = os.environ["RCON_HOST"], 25575, os.environ["RCON_PASSWORD"], sys.argv[1] -def pkt(i, t, b): p = struct.pack("/dev/null; then - cp /src/$f /dst/$f - echo "[wlsync] copied $f" - case $f in - whitelist.json) CMD="whitelist reload" ;; - ops.json) CMD="reload" ;; - banned-players.json|banned-ips.json) CMD="kick @a Ban list updated" ;; - esac - python3 /rcon.py "$CMD" 2>&1 | sed 's/^/[wlsync] rcon: /' || echo "[wlsync] rcon failed" - fi - done - sleep $INTERVAL -done diff --git a/nixos/hosts/asxpio-services/xmpp.nix b/nixos/hosts/asxpio-services/xmpp.nix new file mode 100644 index 0000000..87a9e7e --- /dev/null +++ b/nixos/hosts/asxpio-services/xmpp.nix @@ -0,0 +1,134 @@ +# another.im XMPP server for asxp.io: ejabberd, apushd (push), tackygram +# (Telegram gateway). After compose.yaml and compose.tackygram.yaml of +# https://dev.another.im/another-im/anotherim-server, CERTS=files. +# +# Secrets and keys are host data in /opt/xmpp/state, like a game world, +# and are covered by the restic backup: +# secrets.env ejabberd secrets +# certs/server.pem TLS certificate (asxp.io, conference., xmpp.) +# apushd/ push settings and VAPID key +# tackygram/{env,key} gateway credentials and master key +# Data: Docker volumes xmpp_database, xmpp_tackygram. +{ config, pkgs, ... }: +let + # renovate: datasource=docker depName=dev.another.im/another-im/anotherim-server + ejabberdVersion = "2026.11.07"; + # renovate: datasource=docker depName=dev.another.im/another-im/apushd + apushdVersion = "0.2"; + # renovate: datasource=docker depName=dev.another.im/another-im/tackygram + tackygramVersion = "0.4.3"; + + domain = "asxp.io"; + publicIp = "46.62.228.216"; + state = "/opt/xmpp/state"; + + tcpPorts = [ + 5222 # clients, STARTTLS + 5223 # clients, direct TLS + 5269 # other servers + 5443 # WebSocket, host-meta + 8448 # Matrix federation + 3478 # STUN and TURN + 7777 # SOCKS5 bytestreams proxy + ]; + turnRelay = { + from = 49160; + to = 49199; + }; + gatewayCalls = { + from = 50000; + to = 50099; + }; + range = r: "${toString r.from}-${toString r.to}"; + + needsData.unitConfig.ConditionPathExists = "/opt/.restored"; +in +{ + virtualisation.oci-containers.containers = { + xmpp-ejabberd = { + image = "dev.another.im/another-im/anotherim-server:${ejabberdVersion}"; + environment = { + EJABBERD_MACRO_XMPP_DOMAIN = domain; + EJABBERD_MACRO_TURN_IP = publicIp; + EJABBERD_MACRO_POW_DIFFICULTY = "6"; + EJABBERD_MACRO_REGISTRATION_TIMEOUT = "600"; + EJABBERD_MACRO_UPLOAD_URL = "https://hel1.your-objectstorage.com/asxpio-xmpp-public/"; + EJABBERD_MACRO_ACME_URL = "https://acme-v02.api.letsencrypt.org/directory"; + }; + environmentFiles = [ "${state}/secrets.env" ]; + ports = + map (p: "${toString p}:${toString p}") tcpPorts + ++ [ + "3478:3478/udp" + "${range turnRelay}:${range turnRelay}/udp" + "${range gatewayCalls}:${range gatewayCalls}/udp" + ]; + volumes = [ + "xmpp_database:/opt/ejabberd/database" + "xmpp_logs:/opt/ejabberd/logs" + "${state}/certs:/opt/ejabberd/conf/certs:ro" + ]; + }; + + # Push App Server and Telegram gateway share ejabberd's network + # namespace: they reach its component port on 127.0.0.1 only. They + # restart with it, since the namespace is gone when it stops. + xmpp-apushd = { + image = "dev.another.im/another-im/apushd:${apushdVersion}"; + dependsOn = [ "xmpp-ejabberd" ]; + networks = [ "container:xmpp-ejabberd" ]; + volumes = [ "${state}/apushd:/etc/apushd:ro" ]; + }; + + xmpp-tackygram = { + image = "dev.another.im/another-im/tackygram:${tackygramVersion}"; + dependsOn = [ "xmpp-ejabberd" ]; + networks = [ "container:xmpp-ejabberd" ]; + environmentFiles = [ "${state}/tackygram/env" ]; + environment = { + CREDENTIALS_DIRECTORY = "/run/credentials"; + TACKYGRAM_CALL_IP = publicIp; + TACKYGRAM_CALL_PORTS = range gatewayCalls; + }; + volumes = [ + "xmpp_tackygram:/var/lib/tackygram" + "${state}/tackygram/key:/run/credentials/gateway-key:ro" + ]; + }; + }; + + systemd.services = { + docker-xmpp-ejabberd = needsData; + docker-xmpp-apushd = needsData; + docker-xmpp-tackygram = needsData; + }; + + # For IPv6; see containers.nix + networking.firewall = { + allowedTCPPorts = tcpPorts; + allowedUDPPorts = [ 3478 ]; + allowedUDPPortRanges = [ + turnRelay + gatewayCalls + ]; + }; + + systemd.tmpfiles.rules = [ "d ${state} 0700 root root -" ]; + + # ejabberd's Mnesia through its own backup command, its SQLite through + # the online backup API + asxpio.backup = { + paths = [ + "/var/lib/docker/volumes/xmpp_database" + "/var/lib/docker/volumes/xmpp_tackygram" + ]; + hookPackages = [ + config.virtualisation.docker.package + pkgs.sqlite + ]; + preHooks = [ + "docker exec xmpp-ejabberd ejabberdctl backup /opt/ejabberd/database/nightly.backup" + ''sqlite3 /var/lib/docker/volumes/xmpp_database/_data/ejabberd.sqlite ".backup '$STAGE/ejabberd.sqlite'"'' + ]; + }; +} diff --git a/nixos/hosts/asxpio-services/xray.nix b/nixos/hosts/asxpio-services/xray.nix new file mode 100644 index 0000000..61a8341 --- /dev/null +++ b/nixos/hosts/asxpio-services/xray.nix @@ -0,0 +1,32 @@ +# Xray VLESS Reality proxy on :443, host network. Root in the container +# only to read the 0400 config and bind :443; every other capability is +# dropped. +{ config, ... }: +let + # renovate: datasource=docker depName=ghcr.io/xtls/xray-core + xrayVersion = "26.11.9"; +in +{ + # The whole config.json; a new one restarts the container + sops.secrets."xray/config".restartUnits = [ "docker-xray.service" ]; + + virtualisation.oci-containers.containers.xray = { + image = "ghcr.io/xtls/xray-core:${xrayVersion}"; + cmd = [ + "run" + "-c" + "/etc/xray/config.json" + ]; + user = "0"; + volumes = [ "${config.sops.secrets."xray/config".path}:/etc/xray/config.json:ro" ]; + extraOptions = [ + "--network=host" + "--cap-drop=ALL" + "--cap-add=NET_BIND_SERVICE" + "--security-opt=no-new-privileges:true" + "--read-only" + ]; + }; + + networking.firewall.allowedTCPPorts = [ 443 ]; +} diff --git a/nixos/hosts/rehearsal.nix b/nixos/hosts/rehearsal.nix index ee7d916..e4618f1 100644 --- a/nixos/hosts/rehearsal.nix +++ b/nixos/hosts/rehearsal.nix @@ -1,6 +1,9 @@ # Overrides that turn asxpio-services into a throwaway test machine: # plain DHCP, SSH from anywhere, no WireGuard, dummy secrets. { lib, ... }: +let + manual.autoStart = lib.mkForce false; +in { networking.hostName = "rehearsal"; sops.defaultSopsFile = ../secrets/rehearsal.yaml; @@ -13,11 +16,26 @@ }; networking.wireguard.interfaces = lib.mkForce { }; - systemd.services.podman-sillytavern = { + systemd.services.docker-sillytavern = { after = lib.mkForce [ ]; requires = lib.mkForce [ ]; }; - virtualisation.oci-containers.containers.sillytavern.ports = lib.mkForce [ "127.0.0.1:8000:8000" ]; + virtualisation.oci-containers.containers = { + sillytavern.ports = lib.mkForce [ "127.0.0.1:8000:8000" ]; + + # With production data restored these would act as production: a + # second asxp.io XMPP server and Telegram session, a runner taking + # real CI jobs. Start them by hand, and only with test data. + xmpp-ejabberd = manual; + xmpp-apushd = manual; + xmpp-tackygram = manual; + forgejo-runner = manual; + xray = manual; # its config is a dummy + }; + + # No backups of, or from, a test machine + systemd.timers.restic-backup.enable = false; + systemd.timers.s3-replicate.enable = false; services.openssh.openFirewall = lib.mkForce true; services.openssh.settings.AllowUsers = [ "root" ]; diff --git a/nixos/modules/backup.nix b/nixos/modules/backup.nix new file mode 100644 index 0000000..c57849a --- /dev/null +++ b/nixos/modules/backup.nix @@ -0,0 +1,239 @@ +# Nightly restic backup to Hetzner Object Storage, one repository per +# region, written independently: losing one region or one repository +# leaves the other complete. The same script and command as the Ansible +# role restic_backup installs on asxpio-ops: +# +# restic-backup hel1 snapshots +# restic-backup fsn1 restore latest --target /tmp/r --include /opt/xmpp +{ + config, + lib, + pkgs, + ... +}: +let + cfg = config.asxpio.backup; + host = config.networking.hostName; + + # The release the Ansible role pins (restic_version), so both hosts + # write their repositories with the same version. nixpkgs 26.05 has an + # older one. To bump: change the version, then take the new hash from + # nix store prefetch-file + resticVersion = "0.19.1"; + restic = + pkgs.runCommand "restic-${resticVersion}" + { + src = pkgs.fetchurl { + url = "https://github.com/restic/restic/releases/download/v${resticVersion}/restic_${resticVersion}_linux_amd64.bz2"; + hash = "sha256-9BVBViTcxFLyoCuMM2QXkajG1tO2W7s1Q/z5olFRWFw="; + }; + nativeBuildInputs = [ pkgs.bzip2 ]; + } + '' + mkdir -p $out/bin + bunzip2 -c $src > $out/bin/restic + chmod 755 $out/bin/restic + ''; + + repositories = { + hel1 = "s3:https://hel1.your-objectstorage.com/asxpio-backups-main/restic/${host}"; + fsn1 = "s3:https://fsn1.your-objectstorage.com/asxpio-backups-repl/restic/${host}"; + }; + + envFile = config.sops.templates."restic-env".path; + secret = name: config.sops.placeholder.${name}; + + hooks = lib.imap1 (i: hook: '' + log "hook ${toString i}/${toString (lib.length cfg.preHooks)}" + ( set -e; ${hook} ) || { log "hook failed"; failed=1; } + '') cfg.preHooks; + + backup = pkgs.writeShellApplication { + name = "restic-backup"; + runtimeInputs = [ + restic + pkgs.coreutils + ] + ++ cfg.hookPackages; + # No errexit: a failing hook or repository must not stop the others + bashOptions = [ + "nounset" + "pipefail" + ]; + text = '' + usage() { + cat <<'USAGE' + restic-backup run hooks, then backup + forget to every repository + restic-backup init create the repositories that do not exist yet + restic-backup plain restic against one repository, e.g. + restic-backup hel1 snapshots + restic-backup fsn1 restore latest --target /tmp/r + + Repositories: ${lib.concatStringsSep ", " (lib.attrNames repositories)} + USAGE + } + + set -a + # shellcheck source=/dev/null + . ${envFile} + set +a + export RESTIC_CACHE_DIR=/var/cache/restic + STAGE=${cfg.stageDir} + + declare -A REPOS=( + ${lib.concatStrings (lib.mapAttrsToList (name: url: " [${name}]=\"${url}\"\n") repositories)}) + + log() { echo "restic-backup: $*"; } + + case "''${1:-}" in + init) + for name in "''${!REPOS[@]}"; do + if RESTIC_REPOSITORY="''${REPOS[$name]}" restic cat config >/dev/null 2>&1; then + log "$name: exists" + else + RESTIC_REPOSITORY="''${REPOS[$name]}" restic init || exit 1 + log "$name: created" + fi + done + ;; + + run) + failed=0 + rm -rf "$STAGE" + install -d -m 700 "$STAGE" + export STAGE + ${lib.concatStrings hooks} + for name in "''${!REPOS[@]}"; do + export RESTIC_REPOSITORY="''${REPOS[$name]}" + log "$name: backup" + restic backup --host ${host} --one-file-system --exclude-caches \ + ${lib.concatMapStringsSep " " (e: "--exclude ${lib.escapeShellArg e}") cfg.excludes} \ + ${lib.escapeShellArgs cfg.paths} "$STAGE" || { failed=1; continue; } + + # Prune rewrites pack files, so costs S3 requests: once a week only + prune=() + [ "$(date +%a)" = Sun ] && prune=(--prune) + log "$name: forget ''${prune[*]}" + restic forget --host ${host} --keep-daily 7 --keep-weekly 4 --keep-monthly 6 \ + "''${prune[@]}" || failed=1 + done + + rm -rf "$STAGE" + exit $failed + ;; + + *) + [ -n "''${1:-}" ] && [ -n "''${REPOS[$1]:-}" ] || { usage; exit 2; } + export RESTIC_REPOSITORY="''${REPOS[$1]}" + shift + exec restic "$@" + ;; + esac + ''; + }; + + alert = pkgs.writers.writePython3Bin "restic-alert" { doCheck = false; } ( + builtins.readFile ./restic-alert.py + ); +in +{ + options.asxpio.backup = { + paths = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ + "/etc" + "/opt" + ]; + description = "What to back up, besides the stage directory."; + }; + excludes = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + }; + preHooks = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = '' + Shell lines run in order before the backup, with $STAGE set. A + failing hook marks the run as failed but the backup still runs. + ''; + }; + hookPackages = lib.mkOption { + type = lib.types.listOf lib.types.package; + default = [ ]; + description = "Programs the hooks call."; + }; + stageDir = lib.mkOption { + type = lib.types.str; + default = "/var/backups/restic-stage"; + description = "Dumps written by the hooks land here and are backed up too."; + }; + schedule = lib.mkOption { + type = lib.types.str; + default = "*-*-* 03:30:00"; + }; + }; + + config = { + sops.secrets = { + "restic/password" = { }; + "s3/access_key" = { }; + "s3/secret_key" = { }; + "smtp/user" = { }; + "smtp/password" = { }; + "alert/recipient" = { }; + }; + sops.templates."restic-env".content = '' + RESTIC_PASSWORD=${secret "restic/password"} + AWS_ACCESS_KEY_ID=${secret "s3/access_key"} + AWS_SECRET_ACCESS_KEY=${secret "s3/secret_key"} + ALERT_SMTP_HOST=smtp.fastmail.com + ALERT_SMTP_PORT=587 + ALERT_SMTP_USER=${secret "smtp/user"} + ALERT_SMTP_PASSWORD=${secret "smtp/password"} + ALERT_RECIPIENT=${secret "alert/recipient"} + ''; + + environment.systemPackages = [ backup ]; + systemd.tmpfiles.rules = [ + "d /var/cache/restic 0700 root root -" + "d /var/backups 0755 root root -" + ]; + + systemd.services.restic-backup = { + description = "restic backup to Hetzner Object Storage (${lib.concatStringsSep ", " (lib.attrNames repositories)})"; + wants = [ "network-online.target" ]; + after = [ + "network-online.target" + "docker.service" + ]; + onFailure = [ "restic-alert@%n.service" ]; + serviceConfig = { + Type = "oneshot"; + ExecStart = "${lib.getExe backup} run"; + Nice = 10; + IOSchedulingClass = "idle"; + TimeoutStartSec = "6h"; + }; + }; + systemd.timers.restic-backup = { + description = "Nightly restic backup"; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = cfg.schedule; + RandomizedDelaySec = "10m"; + Persistent = true; + }; + }; + + # Mails the tail of a failed unit's journal, through Forgejo's SMTP + # account. Any unit can use it: OnFailure=restic-alert@%n.service + systemd.services."restic-alert@" = { + description = "Mail the failure of %i"; + serviceConfig = { + Type = "oneshot"; + ExecStart = "${lib.getExe alert} %i ${envFile}"; + }; + }; + }; +} diff --git a/nixos/modules/base.nix b/nixos/modules/base.nix index 7016da5..e4c4571 100644 --- a/nixos/modules/base.nix +++ b/nixos/modules/base.nix @@ -13,7 +13,7 @@ let isNormalUser = true; extraGroups = [ "wheel" - "podman" + "docker" ]; openssh.authorizedKeys.keys = sshKeys; }; diff --git a/nixos/modules/docker.nix b/nixos/modules/docker.nix new file mode 100644 index 0000000..af6d608 --- /dev/null +++ b/nixos/modules/docker.nix @@ -0,0 +1,25 @@ +# Docker, set up as on the Debian hosts. Declared containers become +# systemd units named docker-.service. +{ config, ... }: +{ + virtualisation.docker = { + enable = true; + storageDriver = "overlay2"; + # For containers nobody declared here, such as CI jobs. Declared + # containers log to the journal instead. + logDriver = "json-file"; + daemon.settings.log-opts.max-size = "20m"; + }; + virtualisation.oci-containers.backend = "docker"; + + # Every image update and CI job leaves an image behind + systemd.services.docker-image-prune = { + description = "Remove images no container uses"; + after = [ "docker.service" ]; + requires = [ "docker.service" ]; + path = [ config.virtualisation.docker.package ]; + serviceConfig.Type = "oneshot"; + script = "docker image prune -af"; + startAt = "Mon 03:00"; + }; +} diff --git a/nixos/modules/restic-alert.py b/nixos/modules/restic-alert.py new file mode 100644 index 0000000..1e515da --- /dev/null +++ b/nixos/modules/restic-alert.py @@ -0,0 +1,20 @@ +# Mail the tail of a failed unit's journal. +# Usage: restic-alert +import os, smtplib, socket, subprocess, sys +from email.message import EmailMessage + +unit = sys.argv[1] +env = dict(l.rstrip("\n").split("=", 1) for l in open(sys.argv[2]) if "=" in l) +log = subprocess.run(["journalctl", "-u", unit, "-n", "60", "--no-pager", "-o", "cat"], + capture_output=True, text=True).stdout + +msg = EmailMessage() +msg["Subject"] = f"[{socket.gethostname()}] {unit} failed" +msg["From"] = env["ALERT_SMTP_USER"] +msg["To"] = env["ALERT_RECIPIENT"] +msg.set_content(log) + +with smtplib.SMTP(env["ALERT_SMTP_HOST"], int(env["ALERT_SMTP_PORT"]), timeout=30) as s: + s.starttls() + s.login(env["ALERT_SMTP_USER"], env["ALERT_SMTP_PASSWORD"]) + s.send_message(msg) diff --git a/renovate.json b/renovate.json index 0aefbf6..10094d0 100644 --- a/renovate.json +++ b/renovate.json @@ -1,16 +1,15 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", - "extends": ["config:recommended", ":disableDependencyDashboard"], - "timezone": "Europe/Helsinki", - "labels": ["dependencies"], - "prHourlyLimit": 0, - "prConcurrentLimit": 10, - "commitMessagePrefix": "deps:", + "extends": [ + "local>asxpi/renovate" + ], "customManagers": [ { "description": "Versions in Ansible vars, marked with a comment: # renovate: datasource= depName=", "customType": "regex", - "managerFilePatterns": ["/^ansible/.+\\.ya?ml$/"], + "managerFilePatterns": [ + "/^ansible/.+\\.ya?ml$/" + ], "matchStrings": [ "# renovate: datasource=(?\\S+) depName=(?\\S+)\\s+[a-z_0-9]+:\\s*\"?(?[^\"\\s]+)\"?" ] @@ -19,23 +18,28 @@ "packageRules": [ { "description": "Paper needs Java 21; the tag is a floating JRE line", - "matchPackageNames": ["eclipse-temurin"], + "matchPackageNames": [ + "eclipse-temurin" + ], "enabled": false }, { "description": "Every job runs this image and a nix-store volume tied to its version", - "matchPackageNames": ["nixos/nix"], - "prBodyNotes": ["After merging, remove the old `nix-store-` volume on asxpio-services (see forgejo_runner.yaml)."] + "matchPackageNames": [ + "nixos/nix" + ], + "prBodyNotes": [ + "After merging, remove the old `nix-store-` volume on asxpio-services (see forgejo_runner.yaml)." + ] }, { "description": "A runner change recreates the runner mid-job", - "matchPackageNames": ["code.forgejo.org/forgejo/runner"], - "prBodyNotes": ["The deploy job that applies this shows as cancelled; the next run confirms the result."] - }, - { - "description": "Renovate itself releases daily; once a month is enough", - "matchPackageNames": ["ghcr.io/renovatebot/renovate"], - "schedule": ["* * 1 * *"] + "matchPackageNames": [ + "code.forgejo.org/forgejo/runner" + ], + "prBodyNotes": [ + "The deploy job that applies this shows as cancelled; the next run confirms the result." + ] } ] }