From 8bd9c653f7da885de4a33d8f73831d71c6af6208 Mon Sep 17 00:00:00 2001 From: Sergei Poljanski Date: Thu, 12 Nov 2026 17:00:59 +0400 Subject: [PATCH] nixos: describe asxpio-services as a NixOS system asxpio-services is to move from Debian with Docker to NixOS with Podman. This is the target system; nothing is switched over yet. One flake holds what was spread over cloud-init, Ansible roles and hand-made units: network and WireGuard, users and SSH restrictions, firewall, the weekly image prune, and the four container stacks (Paper and Fossils Minecraft, SillyTavern, the Forgejo runner), each as a systemd unit under Podman. The runner keeps running as a container and starts jobs through Podman's Docker-compatible socket. Secrets (WireGuard keys, RCON and CurseForge keys) are encrypted with sops for the admin's age key and the server's SSH host key, so the repository holds no plaintext secret and CI needs none of them. Containers that own data wait for /opt/.restored. On a freshly installed disk a game server would otherwise generate a new world exactly where the restore is about to land. Tried on a throwaway server (.#rehearsal): install with nixos-anywhere keeping a given host key, secret decryption, all stacks, a real Paper server stopping with a full save, what CI jobs need from the engine (short image names, the nix-store volume, service containers by name, a kaniko build, the socket inside a job), an update over SSH and a reboot. migration/ has the scripts that move the data through object storage, and a workflow builds the system on every push and switches the host on main. Assisted-by: Claude:fable-5-2 --- .forgejo/workflows/nixos.yaml | 65 ++++++++ nixos/.sops.yaml | 13 ++ nixos/README.md | 85 ++++++++++ nixos/flake.lock | 69 ++++++++ nixos/flake.nix | 49 ++++++ nixos/hosts/asxpio-services/containers.nix | 185 +++++++++++++++++++++ nixos/hosts/asxpio-services/default.nix | 85 ++++++++++ nixos/hosts/asxpio-services/runner.nix | 93 +++++++++++ nixos/hosts/asxpio-services/wlsync.sh | 35 ++++ nixos/hosts/rehearsal.nix | 24 +++ nixos/migration/backup-services.sh | 57 +++++++ nixos/migration/restore-services.sh | 41 +++++ nixos/modules/base.nix | 125 ++++++++++++++ nixos/modules/hetzner.nix | 44 +++++ nixos/secrets/asxpio-services.yaml | 29 ++++ nixos/secrets/rehearsal.yaml | 29 ++++ 16 files changed, 1028 insertions(+) create mode 100644 .forgejo/workflows/nixos.yaml create mode 100644 nixos/.sops.yaml create mode 100644 nixos/README.md create mode 100644 nixos/flake.lock create mode 100644 nixos/flake.nix create mode 100644 nixos/hosts/asxpio-services/containers.nix create mode 100644 nixos/hosts/asxpio-services/default.nix create mode 100644 nixos/hosts/asxpio-services/runner.nix create mode 100644 nixos/hosts/asxpio-services/wlsync.sh create mode 100644 nixos/hosts/rehearsal.nix create mode 100755 nixos/migration/backup-services.sh create mode 100755 nixos/migration/restore-services.sh create mode 100644 nixos/modules/base.nix create mode 100644 nixos/modules/hetzner.nix create mode 100644 nixos/secrets/asxpio-services.yaml create mode 100644 nixos/secrets/rehearsal.yaml diff --git a/.forgejo/workflows/nixos.yaml b/.forgejo/workflows/nixos.yaml new file mode 100644 index 0000000..4b031f8 --- /dev/null +++ b/.forgejo/workflows/nixos.yaml @@ -0,0 +1,65 @@ +name: nixos + +# every push: build the system, so a broken config never reaches main +# main: then switch asxpio-services to it +# +# Secret: SSH_DEPLOY_KEY (user deploy). Service secrets are decrypted on +# the server by sops-nix, so CI holds none of them. + +on: + push: + paths: + - 'nixos/**' + - '.forgejo/workflows/nixos.yaml' + workflow_dispatch: + +jobs: + asxpio-services: + runs-on: nix-latest + env: + NIX_CONFIG: | + experimental-features = nix-command flakes + build-users-group = + sandbox = false + NIXPKGS: github:NixOS/nixpkgs/0bb2eedc20e4739f29e8045034f2828ad7054f8f + steps: + - name: Prepare container for actions + run: nix-env -iA nixpkgs.nodejs_22 + + - name: Checkout + uses: actions/checkout@v4 + + - name: Build + working-directory: nixos + run: nix build .#nixosConfigurations.asxpio-services.config.system.build.toplevel --no-link --print-out-paths + + - name: Set up SSH + if: github.ref == 'refs/heads/main' + env: + SSH_DEPLOY_KEY: ${{ secrets.SSH_DEPLOY_KEY }} + run: | + umask 077 + mkdir -p ~/.ssh + printf '%s\n' "$SSH_DEPLOY_KEY" > ~/.ssh/id_ed25519 + cp ansible/known_hosts ~/.ssh/known_hosts + # The runner's own host is reached through asxpio-ops: a job + # container connecting to its host arrives from a bridge + # address that sshd and the firewall refuse. + cat > ~/.ssh/config <<'EOF' + Host asxpio-services + HostName 10.10.0.20 + User deploy + ProxyJump deploy@10.10.0.10 + EOF + + - name: Deploy + if: github.ref == 'refs/heads/main' + working-directory: nixos + run: > + nix shell "$NIXPKGS#nixos-rebuild" "$NIXPKGS#openssh" -c + nixos-rebuild switch --flake .#asxpio-services + --target-host asxpio-services --sudo + + - name: Remove credentials + if: always() + run: rm -f ~/.ssh/id_ed25519 diff --git a/nixos/.sops.yaml b/nixos/.sops.yaml new file mode 100644 index 0000000..d7802ba --- /dev/null +++ b/nixos/.sops.yaml @@ -0,0 +1,13 @@ +# Who can decrypt what. Hosts decrypt with their SSH host key (converted +# to age with ssh-to-age); the admin key edits everything. +keys: + - &admin age19xvmm6j3h2u6mc3w6k65ufry4p5aa5lwu96p55h3e42p9wfzwc0qp2w7pk + - &asxpio_services age15ws8alg3ftsgmzvqxft09556g87s4n8c8v39nmvhaz6k50gp4e7se8acmg + - &rehearsal age1wdgvn9yjjjgqq0h7pysy9e0e74qag4v6t82qyspk2685c550vq3spg0yjj +creation_rules: + - path_regex: secrets/asxpio-services\.yaml$ + key_groups: + - age: [*admin, *asxpio_services] + - path_regex: secrets/rehearsal\.yaml$ + key_groups: + - age: [*admin, *rehearsal] diff --git a/nixos/README.md b/nixos/README.md new file mode 100644 index 0000000..00b7564 --- /dev/null +++ b/nixos/README.md @@ -0,0 +1,85 @@ +# NixOS + +The whole system of a server, as one flake. `asxpio-services` runs +NixOS; `asxpio-ops` is still Debian and managed by `../ansible`. + +``` +flake.nix inputs (nixpkgs 26.05, disko, sops-nix), hosts +modules/base.nix users, SSH, fail2ban, journal cap +modules/hetzner.nix disk layout and boot for a Hetzner Cloud VM +hosts/asxpio-services/ network, WireGuard, firewall, containers, CI runner +hosts/rehearsal.nix overrides for a throwaway test server +secrets/*.yaml sops-encrypted secrets +migration/ one-off scripts for the move from Debian +``` + +Containers run under Podman, one systemd unit each: +`systemctl status podman-minecraft`, `journalctl -u podman-sillytavern`. +`docker` is an alias for `podman`, and `/run/docker.sock` is Podman's +Docker-compatible socket, which the CI runner uses to start jobs. + +## Changing something + +Edit, push a branch (CI builds the system), merge to `main` (CI +deploys). By hand, over WireGuard: + +```sh +nixos-rebuild switch --flake .#asxpio-services --target-host root@10.20.0.20 +``` + +A deploy restarts only the units whose definition changed. Changing a +Minecraft container restarts that server; it gets 120 s to save first. +A bad deploy is undone with `nixos-rebuild switch --rollback` on the +host, or by picking the previous generation in the boot menu. + +To update all packages: `nix flake update`, commit `flake.lock`. + +## Secrets + +`secrets/asxpio-services.yaml` is encrypted with +[sops](https://github.com/getsops/sops) for two keys listed in +`.sops.yaml`: the admin's age key and the server's SSH host key. The +server decrypts at boot into `/run/secrets`. + +```sh +sops secrets/asxpio-services.yaml # edit; needs ~/.config/sops/age/keys.txt +``` + +The server's SSH host key is therefore part of its identity. Keep a copy +outside the server (the `system` archive of a backup contains +`/etc/ssh`), and hand it to a reinstall with `--extra-files`. + +## Installing or reinstalling a server + +Never delete and recreate a production server: it is billed at an old +price that a new server would not get. Reinstall in place. + +1. Put the host key where the installer will copy it from: + + ```sh + mkdir -p extra/etc/ssh # then place ssh_host_ed25519_key{,.pub}, mode 600 + ``` + +2. Install. The running system is replaced; **the disk is wiped**. + + ```sh + nix run github:nix-community/nixos-anywhere -- \ + --flake .#asxpio-services --extra-files extra \ + --ssh-option ProxyJump=root@10.20.0.10 root@10.10.0.20 + ``` + + The installer has no WireGuard, so it is reached over the private + network through asxpio-ops. + +3. Restore data with `migration/restore-services.sh`. Containers that + own data do not start until `/opt/.restored` exists, so nothing + generates a fresh world where the restore is about to land. + +If a server is ever lost entirely, `terraform apply` creates a bare one +and the steps above install onto it. + +## Rehearsal + +`.#rehearsal` is the same system for a throwaway server: plain DHCP, SSH +from anywhere, no WireGuard, dummy secrets. Use it to try an install or +a risky change before production. diff --git a/nixos/flake.lock b/nixos/flake.lock new file mode 100644 index 0000000..d1fc54d --- /dev/null +++ b/nixos/flake.lock @@ -0,0 +1,69 @@ +{ + "nodes": { + "disko": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1794328522, + "narHash": "sha256-h2BYMW4nTFpIqnMd8gwAPFmJAp/1wWLQ6vT0NfL5qt4=", + "owner": "nix-community", + "repo": "disko", + "rev": "1dd30c29c88d99ce295f15e27f93c49bc5fbae07", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "disko", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1794165970, + "narHash": "sha256-XuAV2cUwaod3Bt4XFVqiXz3v1w2ex16W6Sv4S7EWres=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "ac588ef1c914865dad517b337ea0eb9694a4021c", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-26.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "disko": "disko", + "nixpkgs": "nixpkgs", + "sops-nix": "sops-nix" + } + }, + "sops-nix": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1794261845, + "narHash": "sha256-Ivif/kuD68yEOKcZZG3zSwNMFUuIgzUgeJuzcHrQy+8=", + "owner": "Mic92", + "repo": "sops-nix", + "rev": "fe7b4228405aeac2ce7192c2049a6e2bc8637082", + "type": "github" + }, + "original": { + "owner": "Mic92", + "repo": "sops-nix", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/nixos/flake.nix b/nixos/flake.nix new file mode 100644 index 0000000..8c90378 --- /dev/null +++ b/nixos/flake.nix @@ -0,0 +1,49 @@ +{ + description = "asxp.io servers"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-26.05"; + disko = { + url = "github:nix-community/disko"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + sops-nix = { + url = "github:Mic92/sops-nix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + }; + + outputs = + { + nixpkgs, + disko, + sops-nix, + ... + }: + let + mkHost = + modules: + nixpkgs.lib.nixosSystem { + system = "x86_64-linux"; + modules = [ + disko.nixosModules.disko + sops-nix.nixosModules.sops + ./modules/base.nix + ./modules/hetzner.nix + ] + ++ modules; + }; + in + { + nixosConfigurations = { + asxpio-services = mkHost [ ./hosts/asxpio-services ]; + + # The same system on a throwaway server: no WireGuard, no private + # network, dummy secrets. Used to test an install before production. + rehearsal = mkHost [ + ./hosts/asxpio-services + ./hosts/rehearsal.nix + ]; + }; + }; +} diff --git a/nixos/hosts/asxpio-services/containers.nix b/nixos/hosts/asxpio-services/containers.nix new file mode 100644 index 0000000..010ff52 --- /dev/null +++ b/nixos/hosts/asxpio-services/containers.nix @@ -0,0 +1,185 @@ +# Containers run under Podman, one systemd unit each +# (podman-.service). Game data stays in /opt and is not managed here. +{ + config, + lib, + pkgs, + ... +}: +let + podman = lib.getExe pkgs.podman; + # Time for a Minecraft server to save the world before it is killed + stopTimeout = 120; + + # Nothing that owns data starts on a freshly installed disk: a game + # server would generate a new world where the restore is about to land. + # After restoring /opt, `touch /opt/.restored` and start the units. + needsData.unitConfig.ConditionPathExists = "/opt/.restored"; + + gameServer = needsData // { + serviceConfig = { + TimeoutStopSec = lib.mkForce (stopTimeout + 30); + # A JVM stopped by SIGTERM exits 143 after saving; that is a clean stop + SuccessExitStatus = [ 143 ]; + }; + }; +in +{ + virtualisation.podman = { + enable = true; + dockerCompat = true; # `docker` runs podman + dockerSocket.enable = true; # /run/docker.sock, used by the CI runner + defaultNetwork.settings.dns_enabled = true; + autoPrune = { + enable = true; + dates = "Mon 03:00"; + flags = [ "--all" ]; + }; + }; + virtualisation.containers.registries.search = [ "docker.io" ]; + virtualisation.oci-containers.backend = "podman"; + + sops.secrets."minecraft_fossils/env" = { }; + + virtualisation.oci-containers.containers = { + # Paper server; the jar and the world live in /opt/minecraft + minecraft = { + image = "docker.io/library/eclipse-temurin:21-jre"; + workdir = "/data"; + cmd = [ + "java" + "-Xmx4G" + "-Xms2G" + "-jar" + "paper-1.21.11-54.jar" + "nogui" + ]; + ports = [ + "25565:25565" + "19132:19132/udp" + ]; + volumes = [ "/opt/minecraft:/data" ]; + extraOptions = [ + "--tty" + "--interactive" + "--stop-timeout=${toString stopTimeout}" + ]; + }; + + minecraft-fossils = { + image = "docker.io/itzg/minecraft-server:java21"; + environment = { + EULA = "TRUE"; + TYPE = "FABRIC"; + VERSION = "1.20.1"; + FABRIC_LOADER_VERSION = "0.18.4"; + MEMORY = "8G"; + ONLINE_MODE = "FALSE"; + ENFORCE_WHITELIST = "TRUE"; + ENFORCE_SECURE_PROFILE = "FALSE"; + WHITELIST_FILE = "/data/whitelist.json"; + OPS_FILE = "/data/ops.json"; + ENABLE_RCON = "TRUE"; + MOTD = "[mc.czsk.dev] Fossils Archaeology Revival: Genesis"; + USE_AIKAR_FLAGS = "TRUE"; + }; + environmentFiles = [ config.sops.secrets."minecraft_fossils/env".path ]; + ports = [ "25566:25565" ]; + volumes = [ "/opt/minecraft-fossils/data:/data" ]; + extraOptions = [ + "--tty" + "--interactive" + "--network=fossils" + "--stop-timeout=${toString stopTimeout}" + ]; + }; + + # Copies whitelist, ops and ban lists from the Paper server + minecraft-fossils-wlsync = { + image = "docker.io/library/alpine:3"; + dependsOn = [ "minecraft-fossils" ]; + environment = { + RCON_HOST = "minecraft-fossils"; + INTERVAL = "15"; + }; + environmentFiles = [ config.sops.secrets."minecraft_fossils/env".path ]; + volumes = [ + "/opt/minecraft:/src:ro" + "/opt/minecraft-fossils/data:/dst" + "${./wlsync.sh}:/wlsync.sh:ro" + ]; + cmd = [ + "sh" + "/wlsync.sh" + ]; + extraOptions = [ "--network=fossils" ]; + }; + + # Reachable on the WireGuard address only + sillytavern = { + image = "ghcr.io/sillytavern/sillytavern:1.19.0"; + ports = [ "10.20.0.20:8000:8000" ]; + volumes = [ + "/opt/sillytavern/config:/home/node/app/config" + "/opt/sillytavern/data:/home/node/app/data" + "/opt/sillytavern/plugins:/home/node/app/plugins" + ]; + }; + }; + + systemd.services = { + # Network shared by the Fossils server and its sidecar + podman-network-fossils = { + description = "Podman network fossils"; + wantedBy = [ "multi-user.target" ]; + before = [ + "podman-minecraft-fossils.service" + "podman-minecraft-fossils-wlsync.service" + ]; + requiredBy = [ + "podman-minecraft-fossils.service" + "podman-minecraft-fossils-wlsync.service" + ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + }; + script = "${podman} network exists fossils || ${podman} network create fossils"; + }; + + podman-minecraft = gameServer; + podman-minecraft-fossils = gameServer; + podman-minecraft-fossils-wlsync = needsData; + + # The WireGuard address must exist before a port can be bound to it + podman-sillytavern = needsData // { + after = [ "wireguard-asxpio-services.service" ]; + requires = [ "wireguard-asxpio-services.service" ]; + }; + + minecraft-fossils-restart = { + description = "Daily restart of minecraft-fossils"; + serviceConfig.Type = "oneshot"; + script = "systemctl restart podman-minecraft-fossils.service"; + }; + }; + + systemd.timers.minecraft-fossils-restart = { + description = "Restart minecraft-fossils daily at 08:00 Europe/Tallinn"; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = "*-*-* 08:00:00 Europe/Tallinn"; + Persistent = true; + }; + }; + + systemd.tmpfiles.rules = [ + "d /opt 0755 root root -" + "d /opt/minecraft 0755 root root -" + "d /opt/minecraft-fossils 0755 asxpi users -" + "d /opt/minecraft-fossils/data 0755 asxpi users -" + "d /opt/sillytavern/config 0755 root root -" + "d /opt/sillytavern/data 0755 root root -" + "d /opt/sillytavern/plugins 0755 root root -" + ]; +} diff --git a/nixos/hosts/asxpio-services/default.nix b/nixos/hosts/asxpio-services/default.nix new file mode 100644 index 0000000..452520d --- /dev/null +++ b/nixos/hosts/asxpio-services/default.nix @@ -0,0 +1,85 @@ +# asxpio-services: CI runner, Minecraft, SillyTavern. +{ config, lib, ... }: +{ + imports = [ + ./containers.nix + ./runner.nix + ]; + + networking.hostName = lib.mkDefault "asxpio-services"; + networking.domain = "asxp.io"; + system.stateVersion = "26.05"; + + # --- Network ----------------------------------------------------------- + # Interfaces are matched by MAC address: the public one gets IPv4 from + # DHCP and a static IPv6, the private network hands out 10.10.0.20. + networking.useDHCP = false; + networking.useNetworkd = true; + systemd.network.enable = true; + systemd.network.networks = { + "10-public" = { + matchConfig.MACAddress = "92:00:06:b0:4b:85"; + networkConfig.DHCP = "ipv4"; + address = [ "2a01:4f9:c013:8269::1/64" ]; + routes = [ { Gateway = "fe80::1"; } ]; + }; + "20-private" = { + matchConfig.MACAddress = "86:00:00:96:39:d0"; + networkConfig.DHCP = "ipv4"; + dhcpV4Config.RouteMetric = 2000; + linkConfig.RequiredForOnline = "no"; + }; + }; + + # --- WireGuard peer of asxpio-ops --------------------------------------- + sops.defaultSopsFile = lib.mkDefault ../../secrets/asxpio-services.yaml; + sops.age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ]; + sops.secrets."wireguard/private_key" = { }; + sops.secrets."wireguard/preshared_key" = { }; + + networking.wireguard.interfaces.asxpio-services = { + ips = [ + "10.20.0.20/32" + "fd69:69:69::20/128" + ]; + privateKeyFile = config.sops.secrets."wireguard/private_key".path; + peers = [ + { + name = "asxpio-ops"; + publicKey = "0gRrNiV4FjP6ipczZCRD3dsnK7k7rl6IyjG84fPaJB4="; + presharedKeyFile = config.sops.secrets."wireguard/preshared_key".path; + endpoint = "135.181.27.77:60101"; + allowedIPs = [ + "10.20.0.0/24" + "fd69:69:69::/64" + ]; + persistentKeepalive = 25; + } + ]; + }; + + # --- Firewall ----------------------------------------------------------- + # The Hetzner firewall is the outer layer. Published container ports are + # forwarded by Podman and do not pass through these input rules. + networking.firewall = { + allowedTCPPorts = [ + 25565 + 25566 + ]; + allowedUDPPorts = [ 19132 ]; + interfaces = { + asxpio-services.allowedTCPPorts = [ + 22 + 8000 + ]; + enp7s0.allowedTCPPorts = [ 22 ]; + # CI job containers reach the runner's cache server on the host + "podman+".allowedTCPPortRanges = [ + { + from = 1024; + to = 65535; + } + ]; + }; + }; +} diff --git a/nixos/hosts/asxpio-services/runner.nix b/nixos/hosts/asxpio-services/runner.nix new file mode 100644 index 0000000..c69258e --- /dev/null +++ b/nixos/hosts/asxpio-services/runner.nix @@ -0,0 +1,93 @@ +# Forgejo Actions runner. It runs as a container and starts job containers +# through Podman's Docker-compatible socket. Its registration lives in +# /opt/forgejo-runner/data/.runner and is state, not configuration. +{ pkgs, ... }: +let + # Image behind the nix-latest label. Jobs mount a nix-store volume over + # /nix, seeded from this image on first use; the image's /bin/sh and + # /etc/nix/nix.conf link into that store, so a volume from another + # version breaks them. Bumping this also switches to a fresh volume. + nixVersion = "2.35.3"; + + settings = { + log = { + level = "info"; + job_level = "info"; + }; + runner = { + file = "/data/.runner"; + capacity = 4; + envs = { }; + env_file = ".env"; + timeout = "3h"; + shutdown_timeout = "3h"; + insecure = false; + fetch_timeout = "5s"; + fetch_interval = "2s"; + report_interval = "1s"; + labels = [ + "ubuntu-latest:docker://node:20-bookworm" + "ubuntu-22.04:docker://node:20-bookworm" + "ubuntu-24.04:docker://node:22-bookworm" + "debian-latest:docker://debian:bookworm" + "debian-12:docker://debian:bookworm" + "debian-11:docker://debian:bullseye" + "arch-latest:docker://archlinux:latest" + "nginx-latest:docker://nginx:latest" + "nginx-alpine:docker://nginx:alpine" + "node-20:docker://node:20" + "node-22:docker://node:22" + "python-3.11:docker://python:3.11" + "python-3.12:docker://python:3.12" + "alpine-latest:docker://alpine:latest" + "nix-latest:docker://nixos/nix:${nixVersion}" + ]; + }; + cache = { + enabled = true; + port = 0; + dir = ""; + }; + container = { + # "" so each workflow gets a bridge network; services (Postgres) need it + network = ""; + enable_ipv6 = false; + privileged = false; + options = "-v nix-store-${nixVersion}:/nix"; + valid_volumes = [ "**" ]; + docker_host = "automount"; + force_pull = false; + force_rebuild = false; + }; + }; + + configFile = (pkgs.formats.yaml { }).generate "forgejo-runner.yaml" settings; +in +{ + virtualisation.oci-containers.containers.forgejo-runner = { + image = "code.forgejo.org/forgejo/runner:11"; + user = "0:0"; # needs the container socket + cmd = [ + "forgejo-runner" + "daemon" + "--config" + "/config.yaml" + ]; + environment.CONFIG_FILE = "/config.yaml"; + volumes = [ + "/opt/forgejo-runner/data:/data" + "${configFile}:/config.yaml:ro" + "/run/podman/podman.sock:/var/run/docker.sock" + ]; + extraOptions = [ "--network=host" ]; + }; + + systemd.services.podman-forgejo-runner = { + after = [ "podman.socket" ]; + requires = [ "podman.socket" ]; + # Without its registration the runner only crash-loops + unitConfig.ConditionPathExists = "/opt/forgejo-runner/data/.runner"; + }; + + systemd.tmpfiles.rules = [ "d /opt/forgejo-runner/data 0777 root root -" ]; +} diff --git a/nixos/hosts/asxpio-services/wlsync.sh b/nixos/hosts/asxpio-services/wlsync.sh new file mode 100644 index 0000000..6fe4b60 --- /dev/null +++ b/nixos/hosts/asxpio-services/wlsync.sh @@ -0,0 +1,35 @@ +#!/bin/sh +# Mirrors whitelist, ops and ban lists from the Paper server into the +# Fossils server and tells it to reload them over RCON. +apk add --no-cache python3 >/dev/null +cat > /rcon.py <<'PY' +import socket, struct, sys, os +host, port, pw, cmd = os.environ["RCON_HOST"], 25575, os.environ["RCON_PASSWORD"], sys.argv[1] +def pkt(i, t, b): p = struct.pack("/dev/null; then + cp /src/$f /dst/$f + echo "[wlsync] copied $f" + case $f in + whitelist.json) CMD="whitelist reload" ;; + ops.json) CMD="reload" ;; + banned-players.json|banned-ips.json) CMD="kick @a Ban list updated" ;; + esac + python3 /rcon.py "$CMD" 2>&1 | sed 's/^/[wlsync] rcon: /' || echo "[wlsync] rcon failed" + fi + done + sleep $INTERVAL +done diff --git a/nixos/hosts/rehearsal.nix b/nixos/hosts/rehearsal.nix new file mode 100644 index 0000000..ee7d916 --- /dev/null +++ b/nixos/hosts/rehearsal.nix @@ -0,0 +1,24 @@ +# Overrides that turn asxpio-services into a throwaway test machine: +# plain DHCP, SSH from anywhere, no WireGuard, dummy secrets. +{ lib, ... }: +{ + networking.hostName = "rehearsal"; + sops.defaultSopsFile = ../secrets/rehearsal.yaml; + + systemd.network.networks = lib.mkForce { + "10-any" = { + matchConfig.Name = "en* eth*"; + networkConfig.DHCP = "yes"; + }; + }; + + networking.wireguard.interfaces = lib.mkForce { }; + systemd.services.podman-sillytavern = { + after = lib.mkForce [ ]; + requires = lib.mkForce [ ]; + }; + virtualisation.oci-containers.containers.sillytavern.ports = lib.mkForce [ "127.0.0.1:8000:8000" ]; + + services.openssh.openFirewall = lib.mkForce true; + services.openssh.settings.AllowUsers = [ "root" ]; +} diff --git a/nixos/migration/backup-services.sh b/nixos/migration/backup-services.sh new file mode 100755 index 0000000..60d0c75 --- /dev/null +++ b/nixos/migration/backup-services.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# Runs on asxpio-services (Debian). Streams everything worth keeping to +# object storage as tar.zst archives, one per service, with checksums. +# +# backup-services.sh live copy while services run (rehearsal, timing) +# backup-services.sh final stop the game servers and the runner first +# +# Nothing is written to the local disk. Ownership is stored numerically so +# a restore on another OS keeps uid/gid. +set -euo pipefail + +mode=${1:?usage: backup-services.sh live|final} +stamp=$(date -u +%Y%m%dT%H%M%SZ) +dest="asxpio-bucket-hel:asxpio-backups-main/asxpio-services/nixos-migration/$mode-$stamp" +# Hetzner needs the region set and rejects the bucket check rclone does +rc() { rclone --s3-region hel1 --s3-no-check-bucket --s3-chunk-size 64M "$@"; } + +if [ "$mode" = final ]; then + echo "== stopping services" + docker exec minecraft-fossils rcon-cli save-all flush || true + docker stop -t 120 minecraft-fossils minecraft + docker stop -t 30 minecraft-fossils-wlsync sillytavern forgejo-runner +fi + +sums=$(mktemp) +archive() { # name, then paths relative to / + local name=$1 + shift + echo "== $name: $*" + # tar exits 1 when a file changed while being read. That is expected + # for a live copy of a running server and an error in a final one. + { tar --numeric-owner --acls --xattrs -C / -cf - "$@" || { [ $? -eq 1 ] && [ "$mode" = live ]; }; } \ + | zstd -T4 -3 \ + | tee >(sha256sum | sed "s/-\$/$name.tar.zst/" >>"$sums") \ + | rc rcat "$dest/$name.tar.zst" +} + +archive minecraft opt/minecraft +archive minecraft-fossils opt/minecraft-fossils +archive sillytavern opt/sillytavern +archive forgejo-runner opt/forgejo-runner +# Not restored on NixOS; kept in case something in them is wanted later +archive legacy opt/immich opt/wordpress opt/traefik \ + var/lib/docker/volumes/ppeu-wp_wordpress var/lib/docker/volumes/ppeu-wp_wordpress_db +archive system etc root home usr/local var/spool/cron + +sleep 1 +rc rcat "$dest/SHA256SUMS" <"$sums" +echo "== uploaded to $dest" +rc lsl "$dest" +echo "== verifying checksums against the uploaded objects" +while read -r sum name; do + got=$(rc cat "$dest/$name" | sha256sum | cut -d' ' -f1) + [ "$got" = "$sum" ] && echo "ok $name" || { echo "FAIL $name"; exit 1; } +done <"$sums" +rm -f "$sums" +echo "DEST=$dest" diff --git a/nixos/migration/restore-services.sh b/nixos/migration/restore-services.sh new file mode 100755 index 0000000..4827cd6 --- /dev/null +++ b/nixos/migration/restore-services.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Runs on the freshly installed NixOS host. Restores service data from the +# archives written by backup-services.sh, verifies each against SHA256SUMS, +# then lets the containers start. +# +# RCLONE_CONFIG=/run/rclone.conf restore-services.sh [name...] +# +# Without names it restores everything the host needs. The rclone config +# with the object-storage keys is supplied by the operator for the run and +# removed afterwards; it is not part of the system. +set -euo pipefail + +src=${1:?usage: restore-services.sh [name...]} +shift +names=("$@") +[ ${#names[@]} -gt 0 ] || names=(minecraft minecraft-fossils sillytavern forgejo-runner) + +rc() { rclone --s3-region hel1 --s3-no-check-bucket "$@"; } + +sums=$(mktemp) +rc cat "$src/SHA256SUMS" >"$sums" + +for name in "${names[@]}"; do + want=$(awk -v f="$name.tar.zst" '$2 == f {print $1}' "$sums") + [ -n "$want" ] || { echo "no checksum for $name" >&2; exit 1; } + echo "== restoring $name" + got=$(rc cat "$src/$name.tar.zst" \ + | tee >(zstd -d | tar --numeric-owner --acls --xattrs -xpf - -C /) \ + | sha256sum | cut -d' ' -f1) + wait + [ "$got" = "$want" ] || { echo "checksum mismatch for $name" >&2; exit 1; } + echo "ok $name" +done +rm -f "$sums" + +touch /opt/.restored +echo "== data restored; starting containers" +systemctl start --no-block \ + podman-minecraft.service podman-minecraft-fossils.service \ + podman-minecraft-fossils-wlsync.service podman-sillytavern.service \ + podman-forgejo-runner.service diff --git a/nixos/modules/base.nix b/nixos/modules/base.nix new file mode 100644 index 0000000..7016da5 --- /dev/null +++ b/nixos/modules/base.nix @@ -0,0 +1,125 @@ +# Shared by every server: users, SSH, baseline services. +{ lib, pkgs, ... }: +let + keys = { + asxpi-meow = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC//ljrf65xbFyGmy/xeZs5GZvkmufCioqIGYguR3c8c/Ff9HiA4oMRT53Qs16KMuXRg62NusVQvJsvyPRSARpBFsoPiZ/YNBteWlqT1aVqeZwPOfUfFycQ2T8a9qtDnZ0WhamSlJMSAHQPkxXNA6S2ZE9y+dMzMc5qPeRwdsQmuvkxjual/35t9qTgCAG7LvKCKPJkLL5YpwLoqWYWTqSJ6RxNLTYNgIhJBZirzB5rNiFDDupnOuTKSpyRJNxyBSapNge5rCnGQYIeoJ4fi+FTi5wzjlC3hC0BN4Oc/zrm1nS9lyPXQS8KV5/vcQlPrvYZmv0LneeQ/gH/jJsSlwRp2qIatoASSfTlz1y1bOOYtPFojJPikXiXeZMCdlam1II6B94z1ybOIO8TqyKDVs4eHRFfYO72zv0sGsTr7t0C7+N9Wrpmhk38zdlLagZ6HO7KSyoAHm+fQc58fXmOhz4vfYrbmYRPgO0VO03fEb7lSFPPLaSseT0tayli05irJpc= asxpi"; + deploy = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGTDPg3mV55lO8uBnvpcGRxJ2vMRP0qB3Pxhg+C8rwj9 asxpio-deploy"; + zerrtika-workstation = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAB7fAC+8PwqEb1+B523nsdxvUPcTHd3ezHTp9XVe0CP zerrtika@cult-workstation"; + zerrtika-mac = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIELE7hTD4sC5raqjSxy2fXyM+q1CwwVuy9KLFgDR6SFl zerrtika@zerrtika-mac"; + }; + + admin = uid: sshKeys: { + inherit uid; + isNormalUser = true; + extraGroups = [ + "wheel" + "podman" + ]; + openssh.authorizedKeys.keys = sshKeys; + }; + + # Shell access only from WireGuard and the private network + trustedNetworks = [ + "10.20.0.0/24" + "10.10.0.0/20" + ]; + shellUsers = [ + "asxpi" + "zerrtika" + "deploy" + "root" + ]; +in +{ + time.timeZone = "UTC"; + i18n.defaultLocale = "en_US.UTF-8"; + + # Users exist only as declared here + users.mutableUsers = false; + users.users = { + root.openssh.authorizedKeys.keys = [ + keys.asxpi-meow + keys.deploy + keys.zerrtika-workstation + ]; + asxpi = admin 1000 [ keys.asxpi-meow ]; + deploy = admin 1001 [ keys.deploy ]; + zerrtika = admin 1002 [ keys.zerrtika-mac ]; + }; + security.sudo.wheelNeedsPassword = false; + + services.openssh = { + enable = true; + openFirewall = false; # opened per interface by each host + settings = { + PasswordAuthentication = false; + KbdInteractiveAuthentication = false; + PermitRootLogin = "prohibit-password"; + X11Forwarding = false; + MaxAuthTries = 3; + MaxSessions = 5; + ClientAliveInterval = 300; + ClientAliveCountMax = 2; + AllowUsers = lib.mkDefault ( + lib.concatMap (user: map (net: "${user}@${net}") trustedNetworks) shellUsers + ); + Ciphers = [ + "chacha20-poly1305@openssh.com" + "aes256-gcm@openssh.com" + "aes128-gcm@openssh.com" + "aes256-ctr" + "aes192-ctr" + "aes128-ctr" + ]; + Macs = [ + "hmac-sha2-512-etm@openssh.com" + "hmac-sha2-256-etm@openssh.com" + "hmac-sha2-512" + "hmac-sha2-256" + ]; + KexAlgorithms = [ + "sntrup761x25519-sha512@openssh.com" + "curve25519-sha256" + "curve25519-sha256@libssh.org" + "diffie-hellman-group-exchange-sha256" + "diffie-hellman-group16-sha512" + "diffie-hellman-group18-sha512" + ]; + }; + }; + + services.fail2ban.enable = true; + services.journald.extraConfig = "SystemMaxUse=500M"; + + networking.firewall.enable = true; + services.resolved.llmnr = "false"; # no name resolution service on the LAN side + + nix = { + settings = { + experimental-features = [ + "nix-command" + "flakes" + ]; + trusted-users = [ + "root" + "@wheel" + ]; + }; + gc = { + automatic = true; + dates = "weekly"; + options = "--delete-older-than 30d"; + }; + }; + + environment.systemPackages = with pkgs; [ + curl + git + htop + ncdu + rclone + zstd + tree + vim + ]; +} diff --git a/nixos/modules/hetzner.nix b/nixos/modules/hetzner.nix new file mode 100644 index 0000000..bf2a077 --- /dev/null +++ b/nixos/modules/hetzner.nix @@ -0,0 +1,44 @@ +# Hetzner Cloud VM: BIOS boot from /dev/sda, virtio hardware. +{ modulesPath, ... }: +{ + imports = [ (modulesPath + "/profiles/qemu-guest.nix") ]; + + disko.devices.disk.main = { + type = "disk"; + device = "/dev/sda"; + content = { + type = "gpt"; + partitions = { + boot = { + size = "1M"; + type = "EF02"; # BIOS boot, for GRUB + }; + ESP = { + size = "512M"; + type = "EF00"; + content = { + type = "filesystem"; + format = "vfat"; + mountpoint = "/boot"; + }; + }; + root = { + size = "100%"; + content = { + type = "filesystem"; + format = "ext4"; + mountpoint = "/"; + }; + }; + }; + }; + }; + + boot.loader.grub = { + enable = true; + efiSupport = true; + efiInstallAsRemovable = true; + }; + + services.qemuGuest.enable = true; +} diff --git a/nixos/secrets/asxpio-services.yaml b/nixos/secrets/asxpio-services.yaml new file mode 100644 index 0000000..0977381 --- /dev/null +++ b/nixos/secrets/asxpio-services.yaml @@ -0,0 +1,29 @@ +wireguard: + private_key: ENC[AES256_GCM,data:pBLpkt1bpDLSAs061sp3JgE5JTlWoTI6nlDsFngbZrFb1YS2CSszNXpwvEw=,iv:a+pDlqGseDTOAQst7I7zEwq6//bHZLIo33emndeHc+c=,tag:DYJVj/IWNpjXyO8GXJwiWA==,type:str] + preshared_key: ENC[AES256_GCM,data:SubclWuZH2qEqBMsPknnox20hioM4w+JEvj4Dm9HbjibyEpQMGQwnLk9gpc=,iv:OgEVq+j4o1eZVTKQFBsiDGXsf3u6k2fDKFQmHZyrxcw=,tag:5yBdRRrmSzUNCSM++B5Aqg==,type:str] +minecraft_fossils: + env: ENC[AES256_GCM,data:0iMnclZr771sSfcudNhN3ufKBeMVAgDRR6Yk6G3SsI14ZTGv3UYaWks49JroZP2g6Lpv7RQ9f9VU6CYnl6CQxOar9p00GDF81bGWYZKXqtftAwKYof0AiUWoUzDdVvf5rZwb8Qo/vM90kB7+I76D7JdwLXG9PBM=,iv:ruTle2zLOmEvjd1T4upKDhDLVmoPTFiOZfp5pLCtLnI=,tag:Avv1yBeq6WD68qej3SbE7Q==,type:str] +sops: + age: + - enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBCY2E1RW1LRjVFemkyRU8v + TzdjTStiRDkybjZ3NVc3eXk4bHFDeTBzb3hzCndXb1cxd1pJWFBYOWg3ZUdQWlpD + NkRQUE8yd3l2L0ViSmhGQng0SEhVMGcKLS0tIG9KQ2crNUliNWJaaHRXSVlHRmx4 + bmVmajZKSzBsRHBVak9VT2FsM0xoR3MKjQ/TuaPAx/K72DtSlEVsuvwFrD720nf1 + Jco6DWKQo5TX82dII7t/PEQ5/sGT5pSBKkVxRTB81kx8Q944D/iOJA== + -----END AGE ENCRYPTED FILE----- + recipient: age19xvmm6j3h2u6mc3w6k65ufry4p5aa5lwu96p55h3e42p9wfzwc0qp2w7pk + - enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBkNnBwVTJBMVloU1NjcXZN + K01oM09SVklKMXF0OGFZWXFyenZZcjRmRUZBCnI4TktkR3NJOFFIMWs2aUNEcGhR + QWx5Y3NraHVJNHVKemx3NExSenROSVEKLS0tIERsQUdRQ3p4QVNiblhLbmVNOXEy + MjRDVXdkQjI5QjFYczF5RTk3MlloTUUKYRVC+f5iTFufh7LOuqRIz4b+JT/VtRh2 + w93BIK7vvuQrsX/Bshgn4DVgvhTJNmfe86/8c75ZgeyiifkBwI7FMg== + -----END AGE ENCRYPTED FILE----- + recipient: age15ws8alg3ftsgmzvqxft09556g87s4n8c8v39nmvhaz6k50gp4e7se8acmg + lastmodified: "2026-11-12T12:29:11Z" + mac: ENC[AES256_GCM,data:6ZCqGiY2zhtuE87es0xsGGsHuvdAymoZVPnfdvVk3AbK2s3/FKeOteLXSsPxuosMRif7Kfg4kei8Q80g09VW4HpntbpiVLNEIwL6PeSSXIZh6wel/LAZ3ROD8oPC8jAtLB86i7RqOhbPDQvX9DMx90Lx+tyLG2oQsxZUb2podbY=,iv:9UnQ84syvIWeDDbYz5ryv1UAhcrYhref9TeBJw/RheM=,tag:j6qkUwAwvVK0m+DSRWptSQ==,type:str] + unencrypted_suffix: _unencrypted + version: 3.13.3 diff --git a/nixos/secrets/rehearsal.yaml b/nixos/secrets/rehearsal.yaml new file mode 100644 index 0000000..ea8ea93 --- /dev/null +++ b/nixos/secrets/rehearsal.yaml @@ -0,0 +1,29 @@ +wireguard: + private_key: ENC[AES256_GCM,data:W1EAsnxGwL0zVZbaK/9swOQF2w1fiLqBO5Tx9UVCiB2zq0Q7vmjylhZFJmo=,iv:4JykRMiIkaPqaLU3VSgqdmdoSBLm3QNy1G5NhYVhGYU=,tag:rfhIGFD2K7PwqiBJFEa1dQ==,type:str] + preshared_key: ENC[AES256_GCM,data:LyvSVY2lNoB0/TEeB4jTAW8k/rrP+xsiiyEpbe0RCNk468GTiKiqRHXMxUc=,iv:L8ndcxBPBwSDUCZMuoyz2d6tThy7ldiJ0kpYO2odcwI=,tag:4THzv7cMsDBhsWmSlJ0t+w==,type:str] +minecraft_fossils: + env: ENC[AES256_GCM,data:Ofyyf7uidBvn9hURkFbaPtz/iL76uXuo3Yx7zlS/steY6mMmIXzzvo1heKNn,iv:eyJOjcyN4ojMaAApsjbDipv3vPpI2HVS1UGeaW9iZCk=,tag:zoDyalaEazMcB/g6voMZvg==,type:str] +sops: + age: + - enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBWVWRGL1RCeUZqY0ZERkVv + aEowMkE5V2dUd3N3ZXowMkc0RUtrZkFyVlYwCjRQNWszeVN6eVFTOVdiMVRVQnRj + dVNOTFZlNVdIMHVuMGFtUlpaQUhqdW8KLS0tIDRGWm1LUEVvV2lEZTBzTXcycHpl + c25lblJvZmRFdlN5Z1VPRmRNayttNzAKAFhBDdsbpgYImB6ZtnkHm4ffC0R35IJx + 3jIYvz32vnf2hEtBdyfQ+IDIwZso8OXx5aetoG4LjgRk/I8ed0jsjQ== + -----END AGE ENCRYPTED FILE----- + recipient: age19xvmm6j3h2u6mc3w6k65ufry4p5aa5lwu96p55h3e42p9wfzwc0qp2w7pk + - enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBiOGcxZEF6TStSSG1qdEFQ + dldrbXlpSTJJYzh0T0JZazZzelp2dWVITzNRCjU2cmlrdmFqb2xWaEl1d1RqY2N5 + SklpbEIrZkJpZHlLcC9nMWFQNVBOU2sKLS0tIGJEcmZFeE0yNkNEZDU4SUNiTUg1 + UXNyM05BRHVuTEVjT0lMQUp3VnBRdjgKQJDJ1ZMlTM7C7VH1Vt2+D/H69BZmYqv4 + oIGm0ODdeKzfZx8cj/7XT4cmMrbM/wWjqjLzpCAAuX2Rqd4aLbZPqA== + -----END AGE ENCRYPTED FILE----- + recipient: age1wdgvn9yjjjgqq0h7pysy9e0e74qag4v6t82qyspk2685c550vq3spg0yjj + lastmodified: "2026-11-12T12:29:11Z" + mac: ENC[AES256_GCM,data:6qcQN5WIcmvEK3Ywcf8uXyHslU7PM0nFDJTy45ajgenwX5ajay+ZyvO9B/+icKa3bGonIK/K+rUdFPDMvlzfZ+SDdEwD+oQa8JqtNP1h16wauSbZO+9aZuKZBewbCos4FK4/I5PC+48wq87k9MRiM9FsKr/J4H/YfsC3EoGF1XU=,iv:3aBX/UJM8kIeX3mmZ+UFsmw07Xs6Dl/SFOQG9C5ZKLM=,tag:td8XDy0HtClpJbQiKtvbWA==,type:str] + unencrypted_suffix: _unencrypted + version: 3.13.3