nixos: describe asxpio-services as a NixOS system
asxpio-services is to move from Debian with Docker to NixOS with Podman. This is the target system; nothing is switched over yet. One flake holds what was spread over cloud-init, Ansible roles and hand-made units: network and WireGuard, users and SSH restrictions, firewall, the weekly image prune, and the four container stacks (Paper and Fossils Minecraft, SillyTavern, the Forgejo runner), each as a systemd unit under Podman. The runner keeps running as a container and starts jobs through Podman's Docker-compatible socket. Secrets (WireGuard keys, RCON and CurseForge keys) are encrypted with sops for the admin's age key and the server's SSH host key, so the repository holds no plaintext secret and CI needs none of them. Containers that own data wait for /opt/.restored. On a freshly installed disk a game server would otherwise generate a new world exactly where the restore is about to land. Tried on a throwaway server (.#rehearsal): install with nixos-anywhere keeping a given host key, secret decryption, all stacks, a real Paper server stopping with a full save, what CI jobs need from the engine (short image names, the nix-store volume, service containers by name, a kaniko build, the socket inside a job), an update over SSH and a reboot. migration/ has the scripts that move the data through object storage, and a workflow builds the system on every push and switches the host on main. Assisted-by: Claude:fable-5-2
This commit is contained in:
parent
0edfda123a
commit
8d2125ae31
16 changed files with 1028 additions and 0 deletions
185
nixos/hosts/asxpio-services/containers.nix
Normal file
185
nixos/hosts/asxpio-services/containers.nix
Normal file
|
|
@ -0,0 +1,185 @@
|
|||
# Containers run under Podman, one systemd unit each
|
||||
# (podman-<name>.service). Game data stays in /opt and is not managed here.
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
let
|
||||
podman = lib.getExe pkgs.podman;
|
||||
# Time for a Minecraft server to save the world before it is killed
|
||||
stopTimeout = 120;
|
||||
|
||||
# Nothing that owns data starts on a freshly installed disk: a game
|
||||
# server would generate a new world where the restore is about to land.
|
||||
# After restoring /opt, `touch /opt/.restored` and start the units.
|
||||
needsData.unitConfig.ConditionPathExists = "/opt/.restored";
|
||||
|
||||
gameServer = needsData // {
|
||||
serviceConfig = {
|
||||
TimeoutStopSec = lib.mkForce (stopTimeout + 30);
|
||||
# A JVM stopped by SIGTERM exits 143 after saving; that is a clean stop
|
||||
SuccessExitStatus = [ 143 ];
|
||||
};
|
||||
};
|
||||
in
|
||||
{
|
||||
virtualisation.podman = {
|
||||
enable = true;
|
||||
dockerCompat = true; # `docker` runs podman
|
||||
dockerSocket.enable = true; # /run/docker.sock, used by the CI runner
|
||||
defaultNetwork.settings.dns_enabled = true;
|
||||
autoPrune = {
|
||||
enable = true;
|
||||
dates = "Mon 03:00";
|
||||
flags = [ "--all" ];
|
||||
};
|
||||
};
|
||||
virtualisation.containers.registries.search = [ "docker.io" ];
|
||||
virtualisation.oci-containers.backend = "podman";
|
||||
|
||||
sops.secrets."minecraft_fossils/env" = { };
|
||||
|
||||
virtualisation.oci-containers.containers = {
|
||||
# Paper server; the jar and the world live in /opt/minecraft
|
||||
minecraft = {
|
||||
image = "docker.io/library/eclipse-temurin:21-jre";
|
||||
workdir = "/data";
|
||||
cmd = [
|
||||
"java"
|
||||
"-Xmx4G"
|
||||
"-Xms2G"
|
||||
"-jar"
|
||||
"paper-1.21.11-54.jar"
|
||||
"nogui"
|
||||
];
|
||||
ports = [
|
||||
"25565:25565"
|
||||
"19132:19132/udp"
|
||||
];
|
||||
volumes = [ "/opt/minecraft:/data" ];
|
||||
extraOptions = [
|
||||
"--tty"
|
||||
"--interactive"
|
||||
"--stop-timeout=${toString stopTimeout}"
|
||||
];
|
||||
};
|
||||
|
||||
minecraft-fossils = {
|
||||
image = "docker.io/itzg/minecraft-server:java21";
|
||||
environment = {
|
||||
EULA = "TRUE";
|
||||
TYPE = "FABRIC";
|
||||
VERSION = "1.20.1";
|
||||
FABRIC_LOADER_VERSION = "0.18.4";
|
||||
MEMORY = "8G";
|
||||
ONLINE_MODE = "FALSE";
|
||||
ENFORCE_WHITELIST = "TRUE";
|
||||
ENFORCE_SECURE_PROFILE = "FALSE";
|
||||
WHITELIST_FILE = "/data/whitelist.json";
|
||||
OPS_FILE = "/data/ops.json";
|
||||
ENABLE_RCON = "TRUE";
|
||||
MOTD = "[mc.czsk.dev] Fossils Archaeology Revival: Genesis";
|
||||
USE_AIKAR_FLAGS = "TRUE";
|
||||
};
|
||||
environmentFiles = [ config.sops.secrets."minecraft_fossils/env".path ];
|
||||
ports = [ "25566:25565" ];
|
||||
volumes = [ "/opt/minecraft-fossils/data:/data" ];
|
||||
extraOptions = [
|
||||
"--tty"
|
||||
"--interactive"
|
||||
"--network=fossils"
|
||||
"--stop-timeout=${toString stopTimeout}"
|
||||
];
|
||||
};
|
||||
|
||||
# Copies whitelist, ops and ban lists from the Paper server
|
||||
minecraft-fossils-wlsync = {
|
||||
image = "docker.io/library/alpine:3";
|
||||
dependsOn = [ "minecraft-fossils" ];
|
||||
environment = {
|
||||
RCON_HOST = "minecraft-fossils";
|
||||
INTERVAL = "15";
|
||||
};
|
||||
environmentFiles = [ config.sops.secrets."minecraft_fossils/env".path ];
|
||||
volumes = [
|
||||
"/opt/minecraft:/src:ro"
|
||||
"/opt/minecraft-fossils/data:/dst"
|
||||
"${./wlsync.sh}:/wlsync.sh:ro"
|
||||
];
|
||||
cmd = [
|
||||
"sh"
|
||||
"/wlsync.sh"
|
||||
];
|
||||
extraOptions = [ "--network=fossils" ];
|
||||
};
|
||||
|
||||
# Reachable on the WireGuard address only
|
||||
sillytavern = {
|
||||
image = "ghcr.io/sillytavern/sillytavern:1.19.0";
|
||||
ports = [ "10.20.0.20:8000:8000" ];
|
||||
volumes = [
|
||||
"/opt/sillytavern/config:/home/node/app/config"
|
||||
"/opt/sillytavern/data:/home/node/app/data"
|
||||
"/opt/sillytavern/plugins:/home/node/app/plugins"
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services = {
|
||||
# Network shared by the Fossils server and its sidecar
|
||||
podman-network-fossils = {
|
||||
description = "Podman network fossils";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
before = [
|
||||
"podman-minecraft-fossils.service"
|
||||
"podman-minecraft-fossils-wlsync.service"
|
||||
];
|
||||
requiredBy = [
|
||||
"podman-minecraft-fossils.service"
|
||||
"podman-minecraft-fossils-wlsync.service"
|
||||
];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
};
|
||||
script = "${podman} network exists fossils || ${podman} network create fossils";
|
||||
};
|
||||
|
||||
podman-minecraft = gameServer;
|
||||
podman-minecraft-fossils = gameServer;
|
||||
podman-minecraft-fossils-wlsync = needsData;
|
||||
|
||||
# The WireGuard address must exist before a port can be bound to it
|
||||
podman-sillytavern = needsData // {
|
||||
after = [ "wireguard-asxpio-services.service" ];
|
||||
requires = [ "wireguard-asxpio-services.service" ];
|
||||
};
|
||||
|
||||
minecraft-fossils-restart = {
|
||||
description = "Daily restart of minecraft-fossils";
|
||||
serviceConfig.Type = "oneshot";
|
||||
script = "systemctl restart podman-minecraft-fossils.service";
|
||||
};
|
||||
};
|
||||
|
||||
systemd.timers.minecraft-fossils-restart = {
|
||||
description = "Restart minecraft-fossils daily at 08:00 Europe/Tallinn";
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnCalendar = "*-*-* 08:00:00 Europe/Tallinn";
|
||||
Persistent = true;
|
||||
};
|
||||
};
|
||||
|
||||
systemd.tmpfiles.rules = [
|
||||
"d /opt 0755 root root -"
|
||||
"d /opt/minecraft 0755 root root -"
|
||||
"d /opt/minecraft-fossils 0755 asxpi users -"
|
||||
"d /opt/minecraft-fossils/data 0755 asxpi users -"
|
||||
"d /opt/sillytavern/config 0755 root root -"
|
||||
"d /opt/sillytavern/data 0755 root root -"
|
||||
"d /opt/sillytavern/plugins 0755 root root -"
|
||||
];
|
||||
}
|
||||
85
nixos/hosts/asxpio-services/default.nix
Normal file
85
nixos/hosts/asxpio-services/default.nix
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
# asxpio-services: CI runner, Minecraft, SillyTavern.
|
||||
{ config, lib, ... }:
|
||||
{
|
||||
imports = [
|
||||
./containers.nix
|
||||
./runner.nix
|
||||
];
|
||||
|
||||
networking.hostName = lib.mkDefault "asxpio-services";
|
||||
networking.domain = "asxp.io";
|
||||
system.stateVersion = "26.05";
|
||||
|
||||
# --- Network -----------------------------------------------------------
|
||||
# Interfaces are matched by MAC address: the public one gets IPv4 from
|
||||
# DHCP and a static IPv6, the private network hands out 10.10.0.20.
|
||||
networking.useDHCP = false;
|
||||
networking.useNetworkd = true;
|
||||
systemd.network.enable = true;
|
||||
systemd.network.networks = {
|
||||
"10-public" = {
|
||||
matchConfig.MACAddress = "92:00:06:b0:4b:85";
|
||||
networkConfig.DHCP = "ipv4";
|
||||
address = [ "2a01:4f9:c013:8269::1/64" ];
|
||||
routes = [ { Gateway = "fe80::1"; } ];
|
||||
};
|
||||
"20-private" = {
|
||||
matchConfig.MACAddress = "86:00:00:96:39:d0";
|
||||
networkConfig.DHCP = "ipv4";
|
||||
dhcpV4Config.RouteMetric = 2000;
|
||||
linkConfig.RequiredForOnline = "no";
|
||||
};
|
||||
};
|
||||
|
||||
# --- WireGuard peer of asxpio-ops ---------------------------------------
|
||||
sops.defaultSopsFile = lib.mkDefault ../../secrets/asxpio-services.yaml;
|
||||
sops.age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
|
||||
sops.secrets."wireguard/private_key" = { };
|
||||
sops.secrets."wireguard/preshared_key" = { };
|
||||
|
||||
networking.wireguard.interfaces.asxpio-services = {
|
||||
ips = [
|
||||
"10.20.0.20/32"
|
||||
"fd69:69:69::20/128"
|
||||
];
|
||||
privateKeyFile = config.sops.secrets."wireguard/private_key".path;
|
||||
peers = [
|
||||
{
|
||||
name = "asxpio-ops";
|
||||
publicKey = "0gRrNiV4FjP6ipczZCRD3dsnK7k7rl6IyjG84fPaJB4=";
|
||||
presharedKeyFile = config.sops.secrets."wireguard/preshared_key".path;
|
||||
endpoint = "135.181.27.77:60101";
|
||||
allowedIPs = [
|
||||
"10.20.0.0/24"
|
||||
"fd69:69:69::/64"
|
||||
];
|
||||
persistentKeepalive = 25;
|
||||
}
|
||||
];
|
||||
};
|
||||
|
||||
# --- Firewall -----------------------------------------------------------
|
||||
# The Hetzner firewall is the outer layer. Published container ports are
|
||||
# forwarded by Podman and do not pass through these input rules.
|
||||
networking.firewall = {
|
||||
allowedTCPPorts = [
|
||||
25565
|
||||
25566
|
||||
];
|
||||
allowedUDPPorts = [ 19132 ];
|
||||
interfaces = {
|
||||
asxpio-services.allowedTCPPorts = [
|
||||
22
|
||||
8000
|
||||
];
|
||||
enp7s0.allowedTCPPorts = [ 22 ];
|
||||
# CI job containers reach the runner's cache server on the host
|
||||
"podman+".allowedTCPPortRanges = [
|
||||
{
|
||||
from = 1024;
|
||||
to = 65535;
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
}
|
||||
93
nixos/hosts/asxpio-services/runner.nix
Normal file
93
nixos/hosts/asxpio-services/runner.nix
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
# Forgejo Actions runner. It runs as a container and starts job containers
|
||||
# through Podman's Docker-compatible socket. Its registration lives in
|
||||
# /opt/forgejo-runner/data/.runner and is state, not configuration.
|
||||
{ pkgs, ... }:
|
||||
let
|
||||
# Image behind the nix-latest label. Jobs mount a nix-store volume over
|
||||
# /nix, seeded from this image on first use; the image's /bin/sh and
|
||||
# /etc/nix/nix.conf link into that store, so a volume from another
|
||||
# version breaks them. Bumping this also switches to a fresh volume.
|
||||
nixVersion = "2.35.3";
|
||||
|
||||
settings = {
|
||||
log = {
|
||||
level = "info";
|
||||
job_level = "info";
|
||||
};
|
||||
runner = {
|
||||
file = "/data/.runner";
|
||||
capacity = 4;
|
||||
envs = { };
|
||||
env_file = ".env";
|
||||
timeout = "3h";
|
||||
shutdown_timeout = "3h";
|
||||
insecure = false;
|
||||
fetch_timeout = "5s";
|
||||
fetch_interval = "2s";
|
||||
report_interval = "1s";
|
||||
labels = [
|
||||
"ubuntu-latest:docker://node:20-bookworm"
|
||||
"ubuntu-22.04:docker://node:20-bookworm"
|
||||
"ubuntu-24.04:docker://node:22-bookworm"
|
||||
"debian-latest:docker://debian:bookworm"
|
||||
"debian-12:docker://debian:bookworm"
|
||||
"debian-11:docker://debian:bullseye"
|
||||
"arch-latest:docker://archlinux:latest"
|
||||
"nginx-latest:docker://nginx:latest"
|
||||
"nginx-alpine:docker://nginx:alpine"
|
||||
"node-20:docker://node:20"
|
||||
"node-22:docker://node:22"
|
||||
"python-3.11:docker://python:3.11"
|
||||
"python-3.12:docker://python:3.12"
|
||||
"alpine-latest:docker://alpine:latest"
|
||||
"nix-latest:docker://nixos/nix:${nixVersion}"
|
||||
];
|
||||
};
|
||||
cache = {
|
||||
enabled = true;
|
||||
port = 0;
|
||||
dir = "";
|
||||
};
|
||||
container = {
|
||||
# "" so each workflow gets a bridge network; services (Postgres) need it
|
||||
network = "";
|
||||
enable_ipv6 = false;
|
||||
privileged = false;
|
||||
options = "-v nix-store-${nixVersion}:/nix";
|
||||
valid_volumes = [ "**" ];
|
||||
docker_host = "automount";
|
||||
force_pull = false;
|
||||
force_rebuild = false;
|
||||
};
|
||||
};
|
||||
|
||||
configFile = (pkgs.formats.yaml { }).generate "forgejo-runner.yaml" settings;
|
||||
in
|
||||
{
|
||||
virtualisation.oci-containers.containers.forgejo-runner = {
|
||||
image = "code.forgejo.org/forgejo/runner:11";
|
||||
user = "0:0"; # needs the container socket
|
||||
cmd = [
|
||||
"forgejo-runner"
|
||||
"daemon"
|
||||
"--config"
|
||||
"/config.yaml"
|
||||
];
|
||||
environment.CONFIG_FILE = "/config.yaml";
|
||||
volumes = [
|
||||
"/opt/forgejo-runner/data:/data"
|
||||
"${configFile}:/config.yaml:ro"
|
||||
"/run/podman/podman.sock:/var/run/docker.sock"
|
||||
];
|
||||
extraOptions = [ "--network=host" ];
|
||||
};
|
||||
|
||||
systemd.services.podman-forgejo-runner = {
|
||||
after = [ "podman.socket" ];
|
||||
requires = [ "podman.socket" ];
|
||||
# Without its registration the runner only crash-loops
|
||||
unitConfig.ConditionPathExists = "/opt/forgejo-runner/data/.runner";
|
||||
};
|
||||
|
||||
systemd.tmpfiles.rules = [ "d /opt/forgejo-runner/data 0777 root root -" ];
|
||||
}
|
||||
35
nixos/hosts/asxpio-services/wlsync.sh
Normal file
35
nixos/hosts/asxpio-services/wlsync.sh
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
#!/bin/sh
|
||||
# Mirrors whitelist, ops and ban lists from the Paper server into the
|
||||
# Fossils server and tells it to reload them over RCON.
|
||||
apk add --no-cache python3 >/dev/null
|
||||
cat > /rcon.py <<'PY'
|
||||
import socket, struct, sys, os
|
||||
host, port, pw, cmd = os.environ["RCON_HOST"], 25575, os.environ["RCON_PASSWORD"], sys.argv[1]
|
||||
def pkt(i, t, b): p = struct.pack("<ii", i, t) + b.encode() + b"\x00\x00"; return struct.pack("<i", len(p)) + p
|
||||
def recv(s):
|
||||
ln = struct.unpack("<i", s.recv(4))[0]; d = b""
|
||||
while len(d) < ln: d += s.recv(ln - len(d))
|
||||
return struct.unpack("<ii", d[:8])[0], d[8:-2].decode(errors="replace")
|
||||
s = socket.create_connection((host, port), timeout=5)
|
||||
s.send(pkt(1, 3, pw))
|
||||
rid, _ = recv(s)
|
||||
if rid == -1: print("auth failed"); sys.exit(1)
|
||||
s.send(pkt(2, 2, cmd))
|
||||
_, out = recv(s); print(out)
|
||||
PY
|
||||
echo "[wlsync] started, interval=${INTERVAL}s"
|
||||
while true; do
|
||||
for f in whitelist.json ops.json banned-players.json banned-ips.json; do
|
||||
if [ -f /src/$f ] && ! cmp -s /src/$f /dst/$f 2>/dev/null; then
|
||||
cp /src/$f /dst/$f
|
||||
echo "[wlsync] copied $f"
|
||||
case $f in
|
||||
whitelist.json) CMD="whitelist reload" ;;
|
||||
ops.json) CMD="reload" ;;
|
||||
banned-players.json|banned-ips.json) CMD="kick @a Ban list updated" ;;
|
||||
esac
|
||||
python3 /rcon.py "$CMD" 2>&1 | sed 's/^/[wlsync] rcon: /' || echo "[wlsync] rcon failed"
|
||||
fi
|
||||
done
|
||||
sleep $INTERVAL
|
||||
done
|
||||
24
nixos/hosts/rehearsal.nix
Normal file
24
nixos/hosts/rehearsal.nix
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
# Overrides that turn asxpio-services into a throwaway test machine:
|
||||
# plain DHCP, SSH from anywhere, no WireGuard, dummy secrets.
|
||||
{ lib, ... }:
|
||||
{
|
||||
networking.hostName = "rehearsal";
|
||||
sops.defaultSopsFile = ../secrets/rehearsal.yaml;
|
||||
|
||||
systemd.network.networks = lib.mkForce {
|
||||
"10-any" = {
|
||||
matchConfig.Name = "en* eth*";
|
||||
networkConfig.DHCP = "yes";
|
||||
};
|
||||
};
|
||||
|
||||
networking.wireguard.interfaces = lib.mkForce { };
|
||||
systemd.services.podman-sillytavern = {
|
||||
after = lib.mkForce [ ];
|
||||
requires = lib.mkForce [ ];
|
||||
};
|
||||
virtualisation.oci-containers.containers.sillytavern.ports = lib.mkForce [ "127.0.0.1:8000:8000" ];
|
||||
|
||||
services.openssh.openFirewall = lib.mkForce true;
|
||||
services.openssh.settings.AllowUsers = [ "root" ];
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue