diff --git a/.forgejo/workflows/nixos.yaml b/.forgejo/workflows/nixos.yaml new file mode 100644 index 0000000..4b031f8 --- /dev/null +++ b/.forgejo/workflows/nixos.yaml @@ -0,0 +1,65 @@ +name: nixos + +# every push: build the system, so a broken config never reaches main +# main: then switch asxpio-services to it +# +# Secret: SSH_DEPLOY_KEY (user deploy). Service secrets are decrypted on +# the server by sops-nix, so CI holds none of them. + +on: + push: + paths: + - 'nixos/**' + - '.forgejo/workflows/nixos.yaml' + workflow_dispatch: + +jobs: + asxpio-services: + runs-on: nix-latest + env: + NIX_CONFIG: | + experimental-features = nix-command flakes + build-users-group = + sandbox = false + NIXPKGS: github:NixOS/nixpkgs/0bb2eedc20e4739f29e8045034f2828ad7054f8f + steps: + - name: Prepare container for actions + run: nix-env -iA nixpkgs.nodejs_22 + + - name: Checkout + uses: actions/checkout@v4 + + - name: Build + working-directory: nixos + run: nix build .#nixosConfigurations.asxpio-services.config.system.build.toplevel --no-link --print-out-paths + + - name: Set up SSH + if: github.ref == 'refs/heads/main' + env: + SSH_DEPLOY_KEY: ${{ secrets.SSH_DEPLOY_KEY }} + run: | + umask 077 + mkdir -p ~/.ssh + printf '%s\n' "$SSH_DEPLOY_KEY" > ~/.ssh/id_ed25519 + cp ansible/known_hosts ~/.ssh/known_hosts + # The runner's own host is reached through asxpio-ops: a job + # container connecting to its host arrives from a bridge + # address that sshd and the firewall refuse. + cat > ~/.ssh/config <<'EOF' + Host asxpio-services + HostName 10.10.0.20 + User deploy + ProxyJump deploy@10.10.0.10 + EOF + + - name: Deploy + if: github.ref == 'refs/heads/main' + working-directory: nixos + run: > + nix shell "$NIXPKGS#nixos-rebuild" "$NIXPKGS#openssh" -c + nixos-rebuild switch --flake .#asxpio-services + --target-host asxpio-services --sudo + + - name: Remove credentials + if: always() + run: rm -f ~/.ssh/id_ed25519 diff --git a/nixos/.sops.yaml b/nixos/.sops.yaml new file mode 100644 index 0000000..d7802ba --- /dev/null +++ b/nixos/.sops.yaml @@ -0,0 +1,13 @@ +# Who can decrypt what. Hosts decrypt with their SSH host key (converted +# to age with ssh-to-age); the admin key edits everything. +keys: + - &admin age19xvmm6j3h2u6mc3w6k65ufry4p5aa5lwu96p55h3e42p9wfzwc0qp2w7pk + - &asxpio_services age15ws8alg3ftsgmzvqxft09556g87s4n8c8v39nmvhaz6k50gp4e7se8acmg + - &rehearsal age1wdgvn9yjjjgqq0h7pysy9e0e74qag4v6t82qyspk2685c550vq3spg0yjj +creation_rules: + - path_regex: secrets/asxpio-services\.yaml$ + key_groups: + - age: [*admin, *asxpio_services] + - path_regex: secrets/rehearsal\.yaml$ + key_groups: + - age: [*admin, *rehearsal] diff --git a/nixos/README.md b/nixos/README.md new file mode 100644 index 0000000..00b7564 --- /dev/null +++ b/nixos/README.md @@ -0,0 +1,85 @@ +# NixOS + +The whole system of a server, as one flake. `asxpio-services` runs +NixOS; `asxpio-ops` is still Debian and managed by `../ansible`. + +``` +flake.nix inputs (nixpkgs 26.05, disko, sops-nix), hosts +modules/base.nix users, SSH, fail2ban, journal cap +modules/hetzner.nix disk layout and boot for a Hetzner Cloud VM +hosts/asxpio-services/ network, WireGuard, firewall, containers, CI runner +hosts/rehearsal.nix overrides for a throwaway test server +secrets/*.yaml sops-encrypted secrets +migration/ one-off scripts for the move from Debian +``` + +Containers run under Podman, one systemd unit each: +`systemctl status podman-minecraft`, `journalctl -u podman-sillytavern`. +`docker` is an alias for `podman`, and `/run/docker.sock` is Podman's +Docker-compatible socket, which the CI runner uses to start jobs. + +## Changing something + +Edit, push a branch (CI builds the system), merge to `main` (CI +deploys). By hand, over WireGuard: + +```sh +nixos-rebuild switch --flake .#asxpio-services --target-host root@10.20.0.20 +``` + +A deploy restarts only the units whose definition changed. Changing a +Minecraft container restarts that server; it gets 120 s to save first. +A bad deploy is undone with `nixos-rebuild switch --rollback` on the +host, or by picking the previous generation in the boot menu. + +To update all packages: `nix flake update`, commit `flake.lock`. + +## Secrets + +`secrets/asxpio-services.yaml` is encrypted with +[sops](https://github.com/getsops/sops) for two keys listed in +`.sops.yaml`: the admin's age key and the server's SSH host key. The +server decrypts at boot into `/run/secrets`. + +```sh +sops secrets/asxpio-services.yaml # edit; needs ~/.config/sops/age/keys.txt +``` + +The server's SSH host key is therefore part of its identity. Keep a copy +outside the server (the `system` archive of a backup contains +`/etc/ssh`), and hand it to a reinstall with `--extra-files`. + +## Installing or reinstalling a server + +Never delete and recreate a production server: it is billed at an old +price that a new server would not get. Reinstall in place. + +1. Put the host key where the installer will copy it from: + + ```sh + mkdir -p extra/etc/ssh # then place ssh_host_ed25519_key{,.pub}, mode 600 + ``` + +2. Install. The running system is replaced; **the disk is wiped**. + + ```sh + nix run github:nix-community/nixos-anywhere -- \ + --flake .#asxpio-services --extra-files extra \ + --ssh-option ProxyJump=root@10.20.0.10 root@10.10.0.20 + ``` + + The installer has no WireGuard, so it is reached over the private + network through asxpio-ops. + +3. Restore data with `migration/restore-services.sh`. Containers that + own data do not start until `/opt/.restored` exists, so nothing + generates a fresh world where the restore is about to land. + +If a server is ever lost entirely, `terraform apply` creates a bare one +and the steps above install onto it. + +## Rehearsal + +`.#rehearsal` is the same system for a throwaway server: plain DHCP, SSH +from anywhere, no WireGuard, dummy secrets. Use it to try an install or +a risky change before production. diff --git a/nixos/flake.lock b/nixos/flake.lock new file mode 100644 index 0000000..d1fc54d --- /dev/null +++ b/nixos/flake.lock @@ -0,0 +1,69 @@ +{ + "nodes": { + "disko": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1794328522, + "narHash": "sha256-h2BYMW4nTFpIqnMd8gwAPFmJAp/1wWLQ6vT0NfL5qt4=", + "owner": "nix-community", + "repo": "disko", + "rev": "1dd30c29c88d99ce295f15e27f93c49bc5fbae07", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "disko", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1794165970, + "narHash": "sha256-XuAV2cUwaod3Bt4XFVqiXz3v1w2ex16W6Sv4S7EWres=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "ac588ef1c914865dad517b337ea0eb9694a4021c", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-26.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "disko": "disko", + "nixpkgs": "nixpkgs", + "sops-nix": "sops-nix" + } + }, + "sops-nix": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1794261845, + "narHash": "sha256-Ivif/kuD68yEOKcZZG3zSwNMFUuIgzUgeJuzcHrQy+8=", + "owner": "Mic92", + "repo": "sops-nix", + "rev": "fe7b4228405aeac2ce7192c2049a6e2bc8637082", + "type": "github" + }, + "original": { + "owner": "Mic92", + "repo": "sops-nix", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/nixos/flake.nix b/nixos/flake.nix new file mode 100644 index 0000000..8c90378 --- /dev/null +++ b/nixos/flake.nix @@ -0,0 +1,49 @@ +{ + description = "asxp.io servers"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-26.05"; + disko = { + url = "github:nix-community/disko"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + sops-nix = { + url = "github:Mic92/sops-nix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + }; + + outputs = + { + nixpkgs, + disko, + sops-nix, + ... + }: + let + mkHost = + modules: + nixpkgs.lib.nixosSystem { + system = "x86_64-linux"; + modules = [ + disko.nixosModules.disko + sops-nix.nixosModules.sops + ./modules/base.nix + ./modules/hetzner.nix + ] + ++ modules; + }; + in + { + nixosConfigurations = { + asxpio-services = mkHost [ ./hosts/asxpio-services ]; + + # The same system on a throwaway server: no WireGuard, no private + # network, dummy secrets. Used to test an install before production. + rehearsal = mkHost [ + ./hosts/asxpio-services + ./hosts/rehearsal.nix + ]; + }; + }; +} diff --git a/nixos/hosts/asxpio-services/containers.nix b/nixos/hosts/asxpio-services/containers.nix new file mode 100644 index 0000000..010ff52 --- /dev/null +++ b/nixos/hosts/asxpio-services/containers.nix @@ -0,0 +1,185 @@ +# Containers run under Podman, one systemd unit each +# (podman-.service). Game data stays in /opt and is not managed here. +{ + config, + lib, + pkgs, + ... +}: +let + podman = lib.getExe pkgs.podman; + # Time for a Minecraft server to save the world before it is killed + stopTimeout = 120; + + # Nothing that owns data starts on a freshly installed disk: a game + # server would generate a new world where the restore is about to land. + # After restoring /opt, `touch /opt/.restored` and start the units. + needsData.unitConfig.ConditionPathExists = "/opt/.restored"; + + gameServer = needsData // { + serviceConfig = { + TimeoutStopSec = lib.mkForce (stopTimeout + 30); + # A JVM stopped by SIGTERM exits 143 after saving; that is a clean stop + SuccessExitStatus = [ 143 ]; + }; + }; +in +{ + virtualisation.podman = { + enable = true; + dockerCompat = true; # `docker` runs podman + dockerSocket.enable = true; # /run/docker.sock, used by the CI runner + defaultNetwork.settings.dns_enabled = true; + autoPrune = { + enable = true; + dates = "Mon 03:00"; + flags = [ "--all" ]; + }; + }; + virtualisation.containers.registries.search = [ "docker.io" ]; + virtualisation.oci-containers.backend = "podman"; + + sops.secrets."minecraft_fossils/env" = { }; + + virtualisation.oci-containers.containers = { + # Paper server; the jar and the world live in /opt/minecraft + minecraft = { + image = "docker.io/library/eclipse-temurin:21-jre"; + workdir = "/data"; + cmd = [ + "java" + "-Xmx4G" + "-Xms2G" + "-jar" + "paper-1.21.11-54.jar" + "nogui" + ]; + ports = [ + "25565:25565" + "19132:19132/udp" + ]; + volumes = [ "/opt/minecraft:/data" ]; + extraOptions = [ + "--tty" + "--interactive" + "--stop-timeout=${toString stopTimeout}" + ]; + }; + + minecraft-fossils = { + image = "docker.io/itzg/minecraft-server:java21"; + environment = { + EULA = "TRUE"; + TYPE = "FABRIC"; + VERSION = "1.20.1"; + FABRIC_LOADER_VERSION = "0.18.4"; + MEMORY = "8G"; + ONLINE_MODE = "FALSE"; + ENFORCE_WHITELIST = "TRUE"; + ENFORCE_SECURE_PROFILE = "FALSE"; + WHITELIST_FILE = "/data/whitelist.json"; + OPS_FILE = "/data/ops.json"; + ENABLE_RCON = "TRUE"; + MOTD = "[mc.czsk.dev] Fossils Archaeology Revival: Genesis"; + USE_AIKAR_FLAGS = "TRUE"; + }; + environmentFiles = [ config.sops.secrets."minecraft_fossils/env".path ]; + ports = [ "25566:25565" ]; + volumes = [ "/opt/minecraft-fossils/data:/data" ]; + extraOptions = [ + "--tty" + "--interactive" + "--network=fossils" + "--stop-timeout=${toString stopTimeout}" + ]; + }; + + # Copies whitelist, ops and ban lists from the Paper server + minecraft-fossils-wlsync = { + image = "docker.io/library/alpine:3"; + dependsOn = [ "minecraft-fossils" ]; + environment = { + RCON_HOST = "minecraft-fossils"; + INTERVAL = "15"; + }; + environmentFiles = [ config.sops.secrets."minecraft_fossils/env".path ]; + volumes = [ + "/opt/minecraft:/src:ro" + "/opt/minecraft-fossils/data:/dst" + "${./wlsync.sh}:/wlsync.sh:ro" + ]; + cmd = [ + "sh" + "/wlsync.sh" + ]; + extraOptions = [ "--network=fossils" ]; + }; + + # Reachable on the WireGuard address only + sillytavern = { + image = "ghcr.io/sillytavern/sillytavern:1.19.0"; + ports = [ "10.20.0.20:8000:8000" ]; + volumes = [ + "/opt/sillytavern/config:/home/node/app/config" + "/opt/sillytavern/data:/home/node/app/data" + "/opt/sillytavern/plugins:/home/node/app/plugins" + ]; + }; + }; + + systemd.services = { + # Network shared by the Fossils server and its sidecar + podman-network-fossils = { + description = "Podman network fossils"; + wantedBy = [ "multi-user.target" ]; + before = [ + "podman-minecraft-fossils.service" + "podman-minecraft-fossils-wlsync.service" + ]; + requiredBy = [ + "podman-minecraft-fossils.service" + "podman-minecraft-fossils-wlsync.service" + ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + }; + script = "${podman} network exists fossils || ${podman} network create fossils"; + }; + + podman-minecraft = gameServer; + podman-minecraft-fossils = gameServer; + podman-minecraft-fossils-wlsync = needsData; + + # The WireGuard address must exist before a port can be bound to it + podman-sillytavern = needsData // { + after = [ "wireguard-asxpio-services.service" ]; + requires = [ "wireguard-asxpio-services.service" ]; + }; + + minecraft-fossils-restart = { + description = "Daily restart of minecraft-fossils"; + serviceConfig.Type = "oneshot"; + script = "systemctl restart podman-minecraft-fossils.service"; + }; + }; + + systemd.timers.minecraft-fossils-restart = { + description = "Restart minecraft-fossils daily at 08:00 Europe/Tallinn"; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = "*-*-* 08:00:00 Europe/Tallinn"; + Persistent = true; + }; + }; + + systemd.tmpfiles.rules = [ + "d /opt 0755 root root -" + "d /opt/minecraft 0755 root root -" + "d /opt/minecraft-fossils 0755 asxpi users -" + "d /opt/minecraft-fossils/data 0755 asxpi users -" + "d /opt/sillytavern/config 0755 root root -" + "d /opt/sillytavern/data 0755 root root -" + "d /opt/sillytavern/plugins 0755 root root -" + ]; +} diff --git a/nixos/hosts/asxpio-services/default.nix b/nixos/hosts/asxpio-services/default.nix new file mode 100644 index 0000000..452520d --- /dev/null +++ b/nixos/hosts/asxpio-services/default.nix @@ -0,0 +1,85 @@ +# asxpio-services: CI runner, Minecraft, SillyTavern. +{ config, lib, ... }: +{ + imports = [ + ./containers.nix + ./runner.nix + ]; + + networking.hostName = lib.mkDefault "asxpio-services"; + networking.domain = "asxp.io"; + system.stateVersion = "26.05"; + + # --- Network ----------------------------------------------------------- + # Interfaces are matched by MAC address: the public one gets IPv4 from + # DHCP and a static IPv6, the private network hands out 10.10.0.20. + networking.useDHCP = false; + networking.useNetworkd = true; + systemd.network.enable = true; + systemd.network.networks = { + "10-public" = { + matchConfig.MACAddress = "92:00:06:b0:4b:85"; + networkConfig.DHCP = "ipv4"; + address = [ "2a01:4f9:c013:8269::1/64" ]; + routes = [ { Gateway = "fe80::1"; } ]; + }; + "20-private" = { + matchConfig.MACAddress = "86:00:00:96:39:d0"; + networkConfig.DHCP = "ipv4"; + dhcpV4Config.RouteMetric = 2000; + linkConfig.RequiredForOnline = "no"; + }; + }; + + # --- WireGuard peer of asxpio-ops --------------------------------------- + sops.defaultSopsFile = lib.mkDefault ../../secrets/asxpio-services.yaml; + sops.age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ]; + sops.secrets."wireguard/private_key" = { }; + sops.secrets."wireguard/preshared_key" = { }; + + networking.wireguard.interfaces.asxpio-services = { + ips = [ + "10.20.0.20/32" + "fd69:69:69::20/128" + ]; + privateKeyFile = config.sops.secrets."wireguard/private_key".path; + peers = [ + { + name = "asxpio-ops"; + publicKey = "0gRrNiV4FjP6ipczZCRD3dsnK7k7rl6IyjG84fPaJB4="; + presharedKeyFile = config.sops.secrets."wireguard/preshared_key".path; + endpoint = "135.181.27.77:60101"; + allowedIPs = [ + "10.20.0.0/24" + "fd69:69:69::/64" + ]; + persistentKeepalive = 25; + } + ]; + }; + + # --- Firewall ----------------------------------------------------------- + # The Hetzner firewall is the outer layer. Published container ports are + # forwarded by Podman and do not pass through these input rules. + networking.firewall = { + allowedTCPPorts = [ + 25565 + 25566 + ]; + allowedUDPPorts = [ 19132 ]; + interfaces = { + asxpio-services.allowedTCPPorts = [ + 22 + 8000 + ]; + enp7s0.allowedTCPPorts = [ 22 ]; + # CI job containers reach the runner's cache server on the host + "podman+".allowedTCPPortRanges = [ + { + from = 1024; + to = 65535; + } + ]; + }; + }; +} diff --git a/nixos/hosts/asxpio-services/runner.nix b/nixos/hosts/asxpio-services/runner.nix new file mode 100644 index 0000000..c69258e --- /dev/null +++ b/nixos/hosts/asxpio-services/runner.nix @@ -0,0 +1,93 @@ +# Forgejo Actions runner. It runs as a container and starts job containers +# through Podman's Docker-compatible socket. Its registration lives in +# /opt/forgejo-runner/data/.runner and is state, not configuration. +{ pkgs, ... }: +let + # Image behind the nix-latest label. Jobs mount a nix-store volume over + # /nix, seeded from this image on first use; the image's /bin/sh and + # /etc/nix/nix.conf link into that store, so a volume from another + # version breaks them. Bumping this also switches to a fresh volume. + nixVersion = "2.35.3"; + + settings = { + log = { + level = "info"; + job_level = "info"; + }; + runner = { + file = "/data/.runner"; + capacity = 4; + envs = { }; + env_file = ".env"; + timeout = "3h"; + shutdown_timeout = "3h"; + insecure = false; + fetch_timeout = "5s"; + fetch_interval = "2s"; + report_interval = "1s"; + labels = [ + "ubuntu-latest:docker://node:20-bookworm" + "ubuntu-22.04:docker://node:20-bookworm" + "ubuntu-24.04:docker://node:22-bookworm" + "debian-latest:docker://debian:bookworm" + "debian-12:docker://debian:bookworm" + "debian-11:docker://debian:bullseye" + "arch-latest:docker://archlinux:latest" + "nginx-latest:docker://nginx:latest" + "nginx-alpine:docker://nginx:alpine" + "node-20:docker://node:20" + "node-22:docker://node:22" + "python-3.11:docker://python:3.11" + "python-3.12:docker://python:3.12" + "alpine-latest:docker://alpine:latest" + "nix-latest:docker://nixos/nix:${nixVersion}" + ]; + }; + cache = { + enabled = true; + port = 0; + dir = ""; + }; + container = { + # "" so each workflow gets a bridge network; services (Postgres) need it + network = ""; + enable_ipv6 = false; + privileged = false; + options = "-v nix-store-${nixVersion}:/nix"; + valid_volumes = [ "**" ]; + docker_host = "automount"; + force_pull = false; + force_rebuild = false; + }; + }; + + configFile = (pkgs.formats.yaml { }).generate "forgejo-runner.yaml" settings; +in +{ + virtualisation.oci-containers.containers.forgejo-runner = { + image = "code.forgejo.org/forgejo/runner:11"; + user = "0:0"; # needs the container socket + cmd = [ + "forgejo-runner" + "daemon" + "--config" + "/config.yaml" + ]; + environment.CONFIG_FILE = "/config.yaml"; + volumes = [ + "/opt/forgejo-runner/data:/data" + "${configFile}:/config.yaml:ro" + "/run/podman/podman.sock:/var/run/docker.sock" + ]; + extraOptions = [ "--network=host" ]; + }; + + systemd.services.podman-forgejo-runner = { + after = [ "podman.socket" ]; + requires = [ "podman.socket" ]; + # Without its registration the runner only crash-loops + unitConfig.ConditionPathExists = "/opt/forgejo-runner/data/.runner"; + }; + + systemd.tmpfiles.rules = [ "d /opt/forgejo-runner/data 0777 root root -" ]; +} diff --git a/nixos/hosts/asxpio-services/wlsync.sh b/nixos/hosts/asxpio-services/wlsync.sh new file mode 100644 index 0000000..6fe4b60 --- /dev/null +++ b/nixos/hosts/asxpio-services/wlsync.sh @@ -0,0 +1,35 @@ +#!/bin/sh +# Mirrors whitelist, ops and ban lists from the Paper server into the +# Fossils server and tells it to reload them over RCON. +apk add --no-cache python3 >/dev/null +cat > /rcon.py <<'PY' +import socket, struct, sys, os +host, port, pw, cmd = os.environ["RCON_HOST"], 25575, os.environ["RCON_PASSWORD"], sys.argv[1] +def pkt(i, t, b): p = struct.pack("/dev/null; then + cp /src/$f /dst/$f + echo "[wlsync] copied $f" + case $f in + whitelist.json) CMD="whitelist reload" ;; + ops.json) CMD="reload" ;; + banned-players.json|banned-ips.json) CMD="kick @a Ban list updated" ;; + esac + python3 /rcon.py "$CMD" 2>&1 | sed 's/^/[wlsync] rcon: /' || echo "[wlsync] rcon failed" + fi + done + sleep $INTERVAL +done diff --git a/nixos/hosts/rehearsal.nix b/nixos/hosts/rehearsal.nix new file mode 100644 index 0000000..ee7d916 --- /dev/null +++ b/nixos/hosts/rehearsal.nix @@ -0,0 +1,24 @@ +# Overrides that turn asxpio-services into a throwaway test machine: +# plain DHCP, SSH from anywhere, no WireGuard, dummy secrets. +{ lib, ... }: +{ + networking.hostName = "rehearsal"; + sops.defaultSopsFile = ../secrets/rehearsal.yaml; + + systemd.network.networks = lib.mkForce { + "10-any" = { + matchConfig.Name = "en* eth*"; + networkConfig.DHCP = "yes"; + }; + }; + + networking.wireguard.interfaces = lib.mkForce { }; + systemd.services.podman-sillytavern = { + after = lib.mkForce [ ]; + requires = lib.mkForce [ ]; + }; + virtualisation.oci-containers.containers.sillytavern.ports = lib.mkForce [ "127.0.0.1:8000:8000" ]; + + services.openssh.openFirewall = lib.mkForce true; + services.openssh.settings.AllowUsers = [ "root" ]; +} diff --git a/nixos/migration/backup-services.sh b/nixos/migration/backup-services.sh new file mode 100755 index 0000000..60d0c75 --- /dev/null +++ b/nixos/migration/backup-services.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# Runs on asxpio-services (Debian). Streams everything worth keeping to +# object storage as tar.zst archives, one per service, with checksums. +# +# backup-services.sh live copy while services run (rehearsal, timing) +# backup-services.sh final stop the game servers and the runner first +# +# Nothing is written to the local disk. Ownership is stored numerically so +# a restore on another OS keeps uid/gid. +set -euo pipefail + +mode=${1:?usage: backup-services.sh live|final} +stamp=$(date -u +%Y%m%dT%H%M%SZ) +dest="asxpio-bucket-hel:asxpio-backups-main/asxpio-services/nixos-migration/$mode-$stamp" +# Hetzner needs the region set and rejects the bucket check rclone does +rc() { rclone --s3-region hel1 --s3-no-check-bucket --s3-chunk-size 64M "$@"; } + +if [ "$mode" = final ]; then + echo "== stopping services" + docker exec minecraft-fossils rcon-cli save-all flush || true + docker stop -t 120 minecraft-fossils minecraft + docker stop -t 30 minecraft-fossils-wlsync sillytavern forgejo-runner +fi + +sums=$(mktemp) +archive() { # name, then paths relative to / + local name=$1 + shift + echo "== $name: $*" + # tar exits 1 when a file changed while being read. That is expected + # for a live copy of a running server and an error in a final one. + { tar --numeric-owner --acls --xattrs -C / -cf - "$@" || { [ $? -eq 1 ] && [ "$mode" = live ]; }; } \ + | zstd -T4 -3 \ + | tee >(sha256sum | sed "s/-\$/$name.tar.zst/" >>"$sums") \ + | rc rcat "$dest/$name.tar.zst" +} + +archive minecraft opt/minecraft +archive minecraft-fossils opt/minecraft-fossils +archive sillytavern opt/sillytavern +archive forgejo-runner opt/forgejo-runner +# Not restored on NixOS; kept in case something in them is wanted later +archive legacy opt/immich opt/wordpress opt/traefik \ + var/lib/docker/volumes/ppeu-wp_wordpress var/lib/docker/volumes/ppeu-wp_wordpress_db +archive system etc root home usr/local var/spool/cron + +sleep 1 +rc rcat "$dest/SHA256SUMS" <"$sums" +echo "== uploaded to $dest" +rc lsl "$dest" +echo "== verifying checksums against the uploaded objects" +while read -r sum name; do + got=$(rc cat "$dest/$name" | sha256sum | cut -d' ' -f1) + [ "$got" = "$sum" ] && echo "ok $name" || { echo "FAIL $name"; exit 1; } +done <"$sums" +rm -f "$sums" +echo "DEST=$dest" diff --git a/nixos/migration/restore-services.sh b/nixos/migration/restore-services.sh new file mode 100755 index 0000000..4827cd6 --- /dev/null +++ b/nixos/migration/restore-services.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Runs on the freshly installed NixOS host. Restores service data from the +# archives written by backup-services.sh, verifies each against SHA256SUMS, +# then lets the containers start. +# +# RCLONE_CONFIG=/run/rclone.conf restore-services.sh [name...] +# +# Without names it restores everything the host needs. The rclone config +# with the object-storage keys is supplied by the operator for the run and +# removed afterwards; it is not part of the system. +set -euo pipefail + +src=${1:?usage: restore-services.sh [name...]} +shift +names=("$@") +[ ${#names[@]} -gt 0 ] || names=(minecraft minecraft-fossils sillytavern forgejo-runner) + +rc() { rclone --s3-region hel1 --s3-no-check-bucket "$@"; } + +sums=$(mktemp) +rc cat "$src/SHA256SUMS" >"$sums" + +for name in "${names[@]}"; do + want=$(awk -v f="$name.tar.zst" '$2 == f {print $1}' "$sums") + [ -n "$want" ] || { echo "no checksum for $name" >&2; exit 1; } + echo "== restoring $name" + got=$(rc cat "$src/$name.tar.zst" \ + | tee >(zstd -d | tar --numeric-owner --acls --xattrs -xpf - -C /) \ + | sha256sum | cut -d' ' -f1) + wait + [ "$got" = "$want" ] || { echo "checksum mismatch for $name" >&2; exit 1; } + echo "ok $name" +done +rm -f "$sums" + +touch /opt/.restored +echo "== data restored; starting containers" +systemctl start --no-block \ + podman-minecraft.service podman-minecraft-fossils.service \ + podman-minecraft-fossils-wlsync.service podman-sillytavern.service \ + podman-forgejo-runner.service diff --git a/nixos/modules/base.nix b/nixos/modules/base.nix new file mode 100644 index 0000000..7016da5 --- /dev/null +++ b/nixos/modules/base.nix @@ -0,0 +1,125 @@ +# Shared by every server: users, SSH, baseline services. +{ lib, pkgs, ... }: +let + keys = { + asxpi-meow = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC//ljrf65xbFyGmy/xeZs5GZvkmufCioqIGYguR3c8c/Ff9HiA4oMRT53Qs16KMuXRg62NusVQvJsvyPRSARpBFsoPiZ/YNBteWlqT1aVqeZwPOfUfFycQ2T8a9qtDnZ0WhamSlJMSAHQPkxXNA6S2ZE9y+dMzMc5qPeRwdsQmuvkxjual/35t9qTgCAG7LvKCKPJkLL5YpwLoqWYWTqSJ6RxNLTYNgIhJBZirzB5rNiFDDupnOuTKSpyRJNxyBSapNge5rCnGQYIeoJ4fi+FTi5wzjlC3hC0BN4Oc/zrm1nS9lyPXQS8KV5/vcQlPrvYZmv0LneeQ/gH/jJsSlwRp2qIatoASSfTlz1y1bOOYtPFojJPikXiXeZMCdlam1II6B94z1ybOIO8TqyKDVs4eHRFfYO72zv0sGsTr7t0C7+N9Wrpmhk38zdlLagZ6HO7KSyoAHm+fQc58fXmOhz4vfYrbmYRPgO0VO03fEb7lSFPPLaSseT0tayli05irJpc= asxpi"; + deploy = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGTDPg3mV55lO8uBnvpcGRxJ2vMRP0qB3Pxhg+C8rwj9 asxpio-deploy"; + zerrtika-workstation = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAB7fAC+8PwqEb1+B523nsdxvUPcTHd3ezHTp9XVe0CP zerrtika@cult-workstation"; + zerrtika-mac = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIELE7hTD4sC5raqjSxy2fXyM+q1CwwVuy9KLFgDR6SFl zerrtika@zerrtika-mac"; + }; + + admin = uid: sshKeys: { + inherit uid; + isNormalUser = true; + extraGroups = [ + "wheel" + "podman" + ]; + openssh.authorizedKeys.keys = sshKeys; + }; + + # Shell access only from WireGuard and the private network + trustedNetworks = [ + "10.20.0.0/24" + "10.10.0.0/20" + ]; + shellUsers = [ + "asxpi" + "zerrtika" + "deploy" + "root" + ]; +in +{ + time.timeZone = "UTC"; + i18n.defaultLocale = "en_US.UTF-8"; + + # Users exist only as declared here + users.mutableUsers = false; + users.users = { + root.openssh.authorizedKeys.keys = [ + keys.asxpi-meow + keys.deploy + keys.zerrtika-workstation + ]; + asxpi = admin 1000 [ keys.asxpi-meow ]; + deploy = admin 1001 [ keys.deploy ]; + zerrtika = admin 1002 [ keys.zerrtika-mac ]; + }; + security.sudo.wheelNeedsPassword = false; + + services.openssh = { + enable = true; + openFirewall = false; # opened per interface by each host + settings = { + PasswordAuthentication = false; + KbdInteractiveAuthentication = false; + PermitRootLogin = "prohibit-password"; + X11Forwarding = false; + MaxAuthTries = 3; + MaxSessions = 5; + ClientAliveInterval = 300; + ClientAliveCountMax = 2; + AllowUsers = lib.mkDefault ( + lib.concatMap (user: map (net: "${user}@${net}") trustedNetworks) shellUsers + ); + Ciphers = [ + "chacha20-poly1305@openssh.com" + "aes256-gcm@openssh.com" + "aes128-gcm@openssh.com" + "aes256-ctr" + "aes192-ctr" + "aes128-ctr" + ]; + Macs = [ + "hmac-sha2-512-etm@openssh.com" + "hmac-sha2-256-etm@openssh.com" + "hmac-sha2-512" + "hmac-sha2-256" + ]; + KexAlgorithms = [ + "sntrup761x25519-sha512@openssh.com" + "curve25519-sha256" + "curve25519-sha256@libssh.org" + "diffie-hellman-group-exchange-sha256" + "diffie-hellman-group16-sha512" + "diffie-hellman-group18-sha512" + ]; + }; + }; + + services.fail2ban.enable = true; + services.journald.extraConfig = "SystemMaxUse=500M"; + + networking.firewall.enable = true; + services.resolved.llmnr = "false"; # no name resolution service on the LAN side + + nix = { + settings = { + experimental-features = [ + "nix-command" + "flakes" + ]; + trusted-users = [ + "root" + "@wheel" + ]; + }; + gc = { + automatic = true; + dates = "weekly"; + options = "--delete-older-than 30d"; + }; + }; + + environment.systemPackages = with pkgs; [ + curl + git + htop + ncdu + rclone + zstd + tree + vim + ]; +} diff --git a/nixos/modules/hetzner.nix b/nixos/modules/hetzner.nix new file mode 100644 index 0000000..bf2a077 --- /dev/null +++ b/nixos/modules/hetzner.nix @@ -0,0 +1,44 @@ +# Hetzner Cloud VM: BIOS boot from /dev/sda, virtio hardware. +{ modulesPath, ... }: +{ + imports = [ (modulesPath + "/profiles/qemu-guest.nix") ]; + + disko.devices.disk.main = { + type = "disk"; + device = "/dev/sda"; + content = { + type = "gpt"; + partitions = { + boot = { + size = "1M"; + type = "EF02"; # BIOS boot, for GRUB + }; + ESP = { + size = "512M"; + type = "EF00"; + content = { + type = "filesystem"; + format = "vfat"; + mountpoint = "/boot"; + }; + }; + root = { + size = "100%"; + content = { + type = "filesystem"; + format = "ext4"; + mountpoint = "/"; + }; + }; + }; + }; + }; + + boot.loader.grub = { + enable = true; + efiSupport = true; + efiInstallAsRemovable = true; + }; + + services.qemuGuest.enable = true; +} diff --git a/nixos/secrets/asxpio-services.yaml b/nixos/secrets/asxpio-services.yaml new file mode 100644 index 0000000..0977381 --- /dev/null +++ b/nixos/secrets/asxpio-services.yaml @@ -0,0 +1,29 @@ +wireguard: + private_key: ENC[AES256_GCM,data:pBLpkt1bpDLSAs061sp3JgE5JTlWoTI6nlDsFngbZrFb1YS2CSszNXpwvEw=,iv:a+pDlqGseDTOAQst7I7zEwq6//bHZLIo33emndeHc+c=,tag:DYJVj/IWNpjXyO8GXJwiWA==,type:str] + preshared_key: ENC[AES256_GCM,data:SubclWuZH2qEqBMsPknnox20hioM4w+JEvj4Dm9HbjibyEpQMGQwnLk9gpc=,iv:OgEVq+j4o1eZVTKQFBsiDGXsf3u6k2fDKFQmHZyrxcw=,tag:5yBdRRrmSzUNCSM++B5Aqg==,type:str] +minecraft_fossils: + env: ENC[AES256_GCM,data:0iMnclZr771sSfcudNhN3ufKBeMVAgDRR6Yk6G3SsI14ZTGv3UYaWks49JroZP2g6Lpv7RQ9f9VU6CYnl6CQxOar9p00GDF81bGWYZKXqtftAwKYof0AiUWoUzDdVvf5rZwb8Qo/vM90kB7+I76D7JdwLXG9PBM=,iv:ruTle2zLOmEvjd1T4upKDhDLVmoPTFiOZfp5pLCtLnI=,tag:Avv1yBeq6WD68qej3SbE7Q==,type:str] +sops: + age: + - enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBCY2E1RW1LRjVFemkyRU8v + TzdjTStiRDkybjZ3NVc3eXk4bHFDeTBzb3hzCndXb1cxd1pJWFBYOWg3ZUdQWlpD + NkRQUE8yd3l2L0ViSmhGQng0SEhVMGcKLS0tIG9KQ2crNUliNWJaaHRXSVlHRmx4 + bmVmajZKSzBsRHBVak9VT2FsM0xoR3MKjQ/TuaPAx/K72DtSlEVsuvwFrD720nf1 + Jco6DWKQo5TX82dII7t/PEQ5/sGT5pSBKkVxRTB81kx8Q944D/iOJA== + -----END AGE ENCRYPTED FILE----- + recipient: age19xvmm6j3h2u6mc3w6k65ufry4p5aa5lwu96p55h3e42p9wfzwc0qp2w7pk + - enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBkNnBwVTJBMVloU1NjcXZN + K01oM09SVklKMXF0OGFZWXFyenZZcjRmRUZBCnI4TktkR3NJOFFIMWs2aUNEcGhR + QWx5Y3NraHVJNHVKemx3NExSenROSVEKLS0tIERsQUdRQ3p4QVNiblhLbmVNOXEy + MjRDVXdkQjI5QjFYczF5RTk3MlloTUUKYRVC+f5iTFufh7LOuqRIz4b+JT/VtRh2 + w93BIK7vvuQrsX/Bshgn4DVgvhTJNmfe86/8c75ZgeyiifkBwI7FMg== + -----END AGE ENCRYPTED FILE----- + recipient: age15ws8alg3ftsgmzvqxft09556g87s4n8c8v39nmvhaz6k50gp4e7se8acmg + lastmodified: "2026-11-12T12:29:11Z" + mac: ENC[AES256_GCM,data:6ZCqGiY2zhtuE87es0xsGGsHuvdAymoZVPnfdvVk3AbK2s3/FKeOteLXSsPxuosMRif7Kfg4kei8Q80g09VW4HpntbpiVLNEIwL6PeSSXIZh6wel/LAZ3ROD8oPC8jAtLB86i7RqOhbPDQvX9DMx90Lx+tyLG2oQsxZUb2podbY=,iv:9UnQ84syvIWeDDbYz5ryv1UAhcrYhref9TeBJw/RheM=,tag:j6qkUwAwvVK0m+DSRWptSQ==,type:str] + unencrypted_suffix: _unencrypted + version: 3.13.3 diff --git a/nixos/secrets/rehearsal.yaml b/nixos/secrets/rehearsal.yaml new file mode 100644 index 0000000..ea8ea93 --- /dev/null +++ b/nixos/secrets/rehearsal.yaml @@ -0,0 +1,29 @@ +wireguard: + private_key: ENC[AES256_GCM,data:W1EAsnxGwL0zVZbaK/9swOQF2w1fiLqBO5Tx9UVCiB2zq0Q7vmjylhZFJmo=,iv:4JykRMiIkaPqaLU3VSgqdmdoSBLm3QNy1G5NhYVhGYU=,tag:rfhIGFD2K7PwqiBJFEa1dQ==,type:str] + preshared_key: ENC[AES256_GCM,data:LyvSVY2lNoB0/TEeB4jTAW8k/rrP+xsiiyEpbe0RCNk468GTiKiqRHXMxUc=,iv:L8ndcxBPBwSDUCZMuoyz2d6tThy7ldiJ0kpYO2odcwI=,tag:4THzv7cMsDBhsWmSlJ0t+w==,type:str] +minecraft_fossils: + env: ENC[AES256_GCM,data:Ofyyf7uidBvn9hURkFbaPtz/iL76uXuo3Yx7zlS/steY6mMmIXzzvo1heKNn,iv:eyJOjcyN4ojMaAApsjbDipv3vPpI2HVS1UGeaW9iZCk=,tag:zoDyalaEazMcB/g6voMZvg==,type:str] +sops: + age: + - enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBWVWRGL1RCeUZqY0ZERkVv + aEowMkE5V2dUd3N3ZXowMkc0RUtrZkFyVlYwCjRQNWszeVN6eVFTOVdiMVRVQnRj + dVNOTFZlNVdIMHVuMGFtUlpaQUhqdW8KLS0tIDRGWm1LUEVvV2lEZTBzTXcycHpl + c25lblJvZmRFdlN5Z1VPRmRNayttNzAKAFhBDdsbpgYImB6ZtnkHm4ffC0R35IJx + 3jIYvz32vnf2hEtBdyfQ+IDIwZso8OXx5aetoG4LjgRk/I8ed0jsjQ== + -----END AGE ENCRYPTED FILE----- + recipient: age19xvmm6j3h2u6mc3w6k65ufry4p5aa5lwu96p55h3e42p9wfzwc0qp2w7pk + - enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBiOGcxZEF6TStSSG1qdEFQ + dldrbXlpSTJJYzh0T0JZazZzelp2dWVITzNRCjU2cmlrdmFqb2xWaEl1d1RqY2N5 + SklpbEIrZkJpZHlLcC9nMWFQNVBOU2sKLS0tIGJEcmZFeE0yNkNEZDU4SUNiTUg1 + UXNyM05BRHVuTEVjT0lMQUp3VnBRdjgKQJDJ1ZMlTM7C7VH1Vt2+D/H69BZmYqv4 + oIGm0ODdeKzfZx8cj/7XT4cmMrbM/wWjqjLzpCAAuX2Rqd4aLbZPqA== + -----END AGE ENCRYPTED FILE----- + recipient: age1wdgvn9yjjjgqq0h7pysy9e0e74qag4v6t82qyspk2685c550vq3spg0yjj + lastmodified: "2026-11-12T12:29:11Z" + mac: ENC[AES256_GCM,data:6qcQN5WIcmvEK3Ywcf8uXyHslU7PM0nFDJTy45ajgenwX5ajay+ZyvO9B/+icKa3bGonIK/K+rUdFPDMvlzfZ+SDdEwD+oQa8JqtNP1h16wauSbZO+9aZuKZBewbCos4FK4/I5PC+48wq87k9MRiM9FsKr/J4H/YfsC3EoGF1XU=,iv:3aBX/UJM8kIeX3mmZ+UFsmw07Xs6Dl/SFOQG9C5ZKLM=,tag:td8XDy0HtClpJbQiKtvbWA==,type:str] + unencrypted_suffix: _unencrypted + version: 3.13.3