diff --git a/README.md b/README.md index 8993dbc..debbc20 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # asxp.io infrastructure -Two Hetzner Cloud servers, described by Terraform and configured by -Ansible. Pushing to `main` deploys: a Forgejo Actions workflow applies -`ansible/site.yaml` to both servers. +Two Hetzner Cloud servers, described by Terraform. asxpio-services runs +NixOS (`nixos/`); asxpio-ops runs Debian configured by Ansible +(`ansible/`). Pushing to `main` deploys both through Forgejo Actions. ## Servers @@ -17,21 +17,19 @@ Both are reachable for administration only over WireGuard ## Layout - `terraform/`: servers, network, firewalls, object storage buckets. -- `ansible/site.yaml`: everything the pipeline manages (see below). -- `ansible/playbooks/`: host bootstrap, run by hand. -- `.forgejo/workflows/deploy.yaml`: the pipeline. +- `nixos/`: the whole asxpio-services system as a flake. +- `ansible/site.yaml`: everything managed on asxpio-ops. +- `ansible/playbooks/`: asxpio-ops bootstrap, run by hand. +- `.forgejo/workflows/`: `nixos.yaml` and `deploy.yaml`, the pipelines. -## What the pipeline manages +## What is managed where -| Role | Host | What | -|------|------|------| -| `base` | both | unattended upgrades, fail2ban, journal size cap, Docker daemon config, weekly image prune | -| `traefik` | ops | reverse proxy, TLS, dashboard on WireGuard | -| `crawler_alert` | ops | daily and weekly crawler report timers | -| `forgejo` | ops | Forgejo at git.asxp.io (git.czsk.it is an alias), SSH passthrough | -| `forgejo_runner` | services | Actions runner and its job labels | -| `sillytavern` | services | SillyTavern on the WireGuard address | -| `minecraft`, `minecraft_fossils` | services | game servers, daily restart timer | +| What | Host | Where | +|------|------|-------| +| OS, users, SSH, WireGuard, firewall | services | `nixos/` | +| CI runner, Minecraft (Paper, Fossils), SillyTavern | services | `nixos/hosts/asxpio-services/` | +| baseline (upgrades, fail2ban, journal, Docker config) | ops | role `base` | +| Traefik, Forgejo, crawler-alert | ops | roles `traefik`, `forgejo`, `crawler_alert` | Not managed here: the app stacks on asxpio-ops (asxpio, datacenterbingo, blog, czsk, vhpbot, storage). Each deploys from its own repository into @@ -39,37 +37,24 @@ blog, czsk, vhpbot, storage). Each deploys from its own repository into ## Changing something -1. Edit the role or variable, for example an image tag in - `ansible/inventory/group_vars/` or a compose file under - `ansible/roles//files/`. -2. Push a branch. The pipeline does a dry run against production. -3. Merge to `main`. The pipeline dry-runs again, then applies. +Edit, push a branch, merge to `main`. -A run with nothing to change touches nothing and restarts nothing. +- A branch push builds the NixOS system and dry-runs Ansible against + asxpio-ops. +- `main` then switches asxpio-services to the new system and applies + Ansible to asxpio-ops. -Things to know: - -- **Game servers** are saved and stopped with a 120 s timeout before a - compose change recreates them. -- **The runner** executes the pipeline. A change to it recreates the - container mid-job: the change lands, that job is reported as failed, - and the next run confirms it. -- **Forgejo** restarts on a config change; the job log may pause for a - few seconds. - -To run it yourself, over WireGuard: - -```sh -cd ansible -ansible-playbook site.yaml --check --diff # dry run -ansible-playbook site.yaml --tags traefik # one service -``` +A run with nothing to change restarts nothing. A change to the CI +runner restarts the runner under its own job: the change lands, that +job is reported as failed, and the next run confirms it. Details are in +`nixos/README.md` and `ansible/README.md`. ## Secrets -Nothing secret is stored in the clear. Service secrets live in +Nothing secret is stored in the clear. asxpio-services secrets are in +`nixos/secrets/`, encrypted with sops; asxpio-ops secrets are in `ansible/inventory/group_vars/all/vault.yaml`, encrypted with Ansible -Vault. The pipeline needs two repository secrets: +Vault. The pipelines need two repository secrets: - `ANSIBLE_VAULT_PASSWORD`: decrypts the vault. - `SSH_DEPLOY_KEY`: private key of user `deploy` on both servers. @@ -78,6 +63,6 @@ Locally, put the vault password in `ansible/.vault_pass` (ignored by git). ## Bootstrap (by hand) -`ansible/playbooks/` holds what is done once per host or can lock you -out: WireGuard, SSH hardening, s3fs mounts, swap, user creation. -See `ansible/README.md`. +`ansible/playbooks/` holds what is done once on asxpio-ops or can lock +you out: WireGuard, SSH hardening, swap, user creation. Installing or +reinstalling asxpio-services is in `nixos/README.md`. diff --git a/ansible/README.md b/ansible/README.md index e581645..807eb6e 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -1,7 +1,8 @@ # Ansible -`site.yaml` is the desired state of both servers and is applied by the -deploy pipeline on every push to `main`. The playbooks under +`site.yaml` is the desired state of asxpio-ops and is applied by the +deploy pipeline on every push to `main`. asxpio-services runs NixOS and +is described in `../nixos`. The playbooks under `playbooks/` are host bootstrap and are run by hand. ## Running @@ -23,9 +24,10 @@ Templates that carry secrets are never shown in `--diff` output. - `inventory/hosts.yaml`: default. Connects as root to the WireGuard addresses. - `inventory/ci.yaml`: used by the pipeline. Connects as `deploy` over - the private network, reaching asxpio-services through asxpio-ops. + the private network. -`known_hosts` pins both servers' SSH host keys for the pipeline. +`known_hosts` pins both servers' SSH host keys for the pipelines (the +NixOS one uses it too). ## Variables @@ -34,16 +36,12 @@ inventory/ ├── group_vars/all/ │ ├── network.yaml # VPC and WireGuard addresses │ ├── paths.yaml # /opt -│ ├── s3.yaml # object storage buckets and s3fs mounts │ ├── wireguard.yaml # WireGuard peers │ └── vault.yaml # encrypted secrets ├── group_vars/ops_servers/ │ ├── traefik.yaml # traefik_version │ └── forgejo.yaml # forgejo_version, domain and aliases -├── group_vars/services_servers/ -│ ├── forgejo_runner.yaml # runner version, labels' nix image -│ └── minecraft.yaml -└── host_vars/ # per-host addresses, SSH and s3fs settings +└── host_vars/ # per-host addresses and SSH settings ``` Compose files without variables are plain files under @@ -51,8 +49,7 @@ Compose files without variables are plain files under ## Roles in site.yaml -`base`, `traefik`, `crawler_alert`, `forgejo`, `forgejo_runner`, -`sillytavern`, `minecraft`, `minecraft_fossils`. Service roles go +`base`, `traefik`, `crawler_alert`, `forgejo`. Service roles go through `docker_service`, which writes the files, runs `docker compose up -d`, restarts on a config change and waits for the service to answer. @@ -65,22 +62,12 @@ update `roles/crawler_alert/files/` from there. | Playbook | Purpose | |----------|---------| -| `01-wireguard-setup.yaml` | WireGuard hub on asxpio-ops, peer on asxpio-services, client configs | +| `01-wireguard-setup.yaml` | WireGuard hub on asxpio-ops, client configs | | `02-ssh-hardening.yaml` | restrict SSH to WireGuard and the private network | -| `03-setup-s3fs.yaml` | s3fs mounts on asxpio-services (off on asxpio-ops) | | `04-add-user-zerrtika.yaml` | add a user to running servers | | `05-swap.yaml` | 2 GB swapfile on asxpio-ops | Always dry-run these first with `--check --diff`. -- `03-setup-s3fs.yaml` rewrites `/etc/s3fs/.passwd-s3fs`, which the - backup script on asxpio-ops also reads. -- On asxpio-services the WireGuard interface is named - `asxpio-services`, not `wg0`. Check `01-wireguard-setup.yaml` against - the host before re-running it there. - -## Runner notes - -The `nix-latest` label runs `nixos/nix:` -with a `nix-store-` volume over `/nix`. To bump it: change the -variable, let the pipeline apply it, then remove the old volume. +`/etc/s3fs/.passwd-s3fs` on asxpio-ops is a leftover of the removed s3fs +role and is still read by the backup script there; nothing manages it. diff --git a/ansible/inventory/ci.yaml b/ansible/inventory/ci.yaml index 3d8d2fa..78af873 100644 --- a/ansible/inventory/ci.yaml +++ b/ansible/inventory/ci.yaml @@ -1,9 +1,6 @@ --- -# Inventory for the Forgejo Actions runner. Jobs run in containers on -# asxpio-services, so asxpio-ops is reached over the VPC. The services -# host itself is reached through asxpio-ops: a container talking to its -# own host arrives from a Docker bridge address, which sshd and the -# firewall both refuse. +# Inventory for the Forgejo Actions runner: asxpio-ops over the private +# network, as user deploy. all: vars: @@ -17,8 +14,3 @@ all: hosts: asxpio-ops: ansible_host: "{{ vpc_ip }}" - services_servers: - hosts: - asxpio-services: - ansible_host: "{{ vpc_ip }}" - ansible_ssh_common_args: "-o ProxyJump=deploy@{{ asxpio_ops_vpc_ip }}" diff --git a/ansible/inventory/group_vars/all/s3.yaml b/ansible/inventory/group_vars/all/s3.yaml deleted file mode 100644 index 852ff4c..0000000 --- a/ansible/inventory/group_vars/all/s3.yaml +++ /dev/null @@ -1,39 +0,0 @@ ---- -# S3 Object Storage Configuration -# These should be overridden in vault or environment-specific vars - -# S3 Endpoint Configuration -s3_region: hel1 -s3_endpoint: "https://{{ s3_region }}.your-objectstorage.com" - -# S3 Credentials -# WARNING: Store these in ansible-vault encrypted file! -# Override these in group_vars/all/vault.yaml (encrypted) -s3_access_key: "{{ vault_s3_access_key | default('') }}" -s3_secret_key: "{{ vault_s3_secret_key | default('') }}" - -# S3 Bucket names -s3_bucket_terraform_main: asxpio-terraform-main -s3_bucket_terraform_repl: asxpio-terraform-repl -s3_bucket_backups_main: asxpio-backups-main -s3_bucket_backups_repl: asxpio-backups-repl -s3_bucket_immich_main: asxpio-immich-main -s3_bucket_immich_repl: asxpio-immich-repl - -# S3 Buckets to mount on servers (non-replication only) -s3_buckets: - - name: "{{ s3_bucket_terraform_main }}" - mount_point: /mnt/s3/terraform - owner: root - group: root - mode: "0755" - - name: "{{ s3_bucket_backups_main }}" - mount_point: /mnt/s3/backups - owner: root - group: root - mode: "0755" - - name: "{{ s3_bucket_immich_main }}" - mount_point: /mnt/s3/immich - owner: root - group: docker - mode: "0775" diff --git a/ansible/inventory/group_vars/services_servers/forgejo_runner.yaml b/ansible/inventory/group_vars/services_servers/forgejo_runner.yaml deleted file mode 100644 index 93b47c4..0000000 --- a/ansible/inventory/group_vars/services_servers/forgejo_runner.yaml +++ /dev/null @@ -1,20 +0,0 @@ ---- -# Forgejo Runner (CI/CD) configuration - -# Runner version and settings -forgejo_runner_version: "11" -forgejo_runner_capacity: 4 -forgejo_runner_timeout: 3h - -# Image behind the nix-latest label. Jobs mount a nix-store volume over -# /nix, seeded from this image on first use; the image's /bin/sh and -# /etc/nix/nix.conf link into that store, so a volume from another -# version breaks them. Bumping this also switches to a fresh volume. -forgejo_runner_nix_version: "2.35.3" -forgejo_runner_token: "{{ vault_forgejo_runner_token }}" - -# Runner installation path -forgejo_runner_base_path: "{{ services_base_path }}/forgejo-runner" - -# The runner talks to Forgejo over the VPC, not through Traefik -forgejo_http_port: 3000 diff --git a/ansible/inventory/group_vars/services_servers/minecraft.yaml b/ansible/inventory/group_vars/services_servers/minecraft.yaml deleted file mode 100644 index 6914552..0000000 --- a/ansible/inventory/group_vars/services_servers/minecraft.yaml +++ /dev/null @@ -1,4 +0,0 @@ ---- -minecraft_fossils_owner: asxpi -minecraft_fossils_rcon_password: "{{ vault_minecraft_fossils_rcon_password }}" -minecraft_fossils_cf_api_key: "{{ vault_minecraft_fossils_cf_api_key }}" diff --git a/ansible/inventory/host_vars/asxpio-ops.yaml b/ansible/inventory/host_vars/asxpio-ops.yaml index 18a613a..721582b 100644 --- a/ansible/inventory/host_vars/asxpio-ops.yaml +++ b/ansible/inventory/host_vars/asxpio-ops.yaml @@ -6,7 +6,3 @@ wireguard_ip: "{{ asxpio_ops_wireguard_ip }}" # SSH: the git user may connect from anywhere (Forgejo passthrough) ssh_allow_public_git: true ssh_git_user: git - -# s3fs is off here: its ensure_diskfree reserve never fit the 38 GB disk -s3fs_enabled: false -s3fs_ensure_diskfree: 20000 diff --git a/ansible/inventory/host_vars/asxpio-services.yaml b/ansible/inventory/host_vars/asxpio-services.yaml deleted file mode 100644 index 02be751..0000000 --- a/ansible/inventory/host_vars/asxpio-services.yaml +++ /dev/null @@ -1,8 +0,0 @@ ---- -public_ip: 46.62.228.216 -vpc_ip: "{{ asxpio_services_vpc_ip }}" -wireguard_ip: "{{ asxpio_services_wireguard_ip }}" - -ssh_allow_public_git: false - -s3fs_ensure_diskfree: 81920 # 80 GB free = max 80 GB cache diff --git a/ansible/inventory/hosts.yaml b/ansible/inventory/hosts.yaml index 685ce17..69b4f48 100644 --- a/ansible/inventory/hosts.yaml +++ b/ansible/inventory/hosts.yaml @@ -1,6 +1,7 @@ --- # Default inventory: run from a workstation over WireGuard, as root. # CI uses inventory/ci.yaml instead (VPC addresses, user deploy). +# asxpio-services runs NixOS and is not managed by Ansible; see ../nixos. all: vars: @@ -14,7 +15,3 @@ all: hosts: asxpio-ops: ansible_host: "{{ wireguard_ip }}" - services_servers: - hosts: - asxpio-services: - ansible_host: "{{ wireguard_ip }}" diff --git a/ansible/playbooks/01-wireguard-setup.yaml b/ansible/playbooks/01-wireguard-setup.yaml index a410c00..bd9f2a6 100644 --- a/ansible/playbooks/01-wireguard-setup.yaml +++ b/ansible/playbooks/01-wireguard-setup.yaml @@ -43,73 +43,6 @@ roles: - role: wireguard -- name: Deploy WireGuard VPN Client - hosts: services_servers - become: true - gather_facts: true - - vars: - # Map each server to its client config name - wireguard_client_configs: - asxpio-services: asxpio-services - - tasks: - - name: Display client deployment information - ansible.builtin.debug: - msg: | - ======================================== - WireGuard VPN Client Deployment - ======================================== - Target: {{ inventory_hostname }} (VPN Client) - Will connect to: {{ asxpio_ops_wireguard_ip }} (asxpio-ops) - ======================================== - - - name: Get client config name for this host - ansible.builtin.set_fact: - client_config_name: "{{ wireguard_client_configs[inventory_hostname] }}" - when: inventory_hostname in wireguard_client_configs - - - name: Install WireGuard - ansible.builtin.apt: - name: - - wireguard - - wireguard-tools - state: present - update_cache: true - - - name: Create WireGuard config directory - ansible.builtin.file: - path: /etc/wireguard - state: directory - mode: '0700' - owner: root - group: root - - - name: Copy WireGuard client configuration - ansible.builtin.copy: - src: "{{ playbook_dir }}/wireguard-configs/asxpio-ops/{{ client_config_name }}.conf" - dest: "/etc/wireguard/{{ client_config_name }}.conf" - mode: '0600' - owner: root - group: root - when: client_config_name is defined - - - name: Enable and start WireGuard interface - ansible.builtin.systemd: - name: "wg-quick@{{ client_config_name }}" - enabled: true - state: started - when: client_config_name is defined - register: wg_client_start - - - name: Display client status - ansible.builtin.debug: - msg: | - ✅ WireGuard client configured on {{ inventory_hostname }} - Interface: wg-quick@{{ client_config_name }} - Status: {{ 'Started' if wg_client_start.changed else 'Already running' }} - when: client_config_name is defined - - name: Post-deployment verification hosts: localhost gather_facts: false @@ -123,10 +56,9 @@ Infrastructure deployed: ✅ asxpio-ops: WireGuard VPN Server (hub) - ✅ asxpio-services: WireGuard VPN Client (connected to ops) Server-to-server connectivity: - - asxpio-ops ↔ asxpio-services via WireGuard (10.20.0.0/24) + - asxpio-services is a peer too; its side is configured in ../nixos IMPORTANT NEXT STEPS FOR YOUR WORKSTATION: diff --git a/ansible/playbooks/03-setup-s3fs.yaml b/ansible/playbooks/03-setup-s3fs.yaml deleted file mode 100644 index 73e95ef..0000000 --- a/ansible/playbooks/03-setup-s3fs.yaml +++ /dev/null @@ -1,134 +0,0 @@ ---- -# Stage 3: Setup S3FS on all servers -# -# This playbook sets up s3fs to mount S3 buckets as local filesystems. -# Must be run after Stage 2 (full deployment) to ensure Docker is installed. -# -# Mounts the following buckets: -# - asxpio-terraform-main -> /mnt/s3/terraform -# - asxpio-backups-main -> /mnt/s3/backups -# - asxpio-immich-main -> /mnt/s3/immich -# -# Cache sizes (per-host): -# - asxpio-ops (40GB disk): 10GB cache max -# - asxpio-services (160GB disk): 80GB cache max -# -# Usage: -# ansible-playbook playbooks/03-setup-s3fs.yaml -# -# IMPORTANT: Ensure S3 credentials are set in group_vars/all/vault.yaml: -# vault_s3_access_key: "your-access-key" -# vault_s3_secret_key: "your-secret-key" -# - -- name: Setup S3FS on all servers - hosts: ops_servers,services_servers - become: true - gather_facts: true - - pre_tasks: - - name: Validate S3 credentials are set - ansible.builtin.assert: - that: - - s3_access_key is defined - - s3_access_key != "" - - s3_secret_key is defined - - s3_secret_key != "" - fail_msg: | - S3 credentials not set! - Please set vault_s3_access_key and vault_s3_secret_key in: - - group_vars/all/vault.yaml (encrypted with ansible-vault) - OR - - Pass via command line: -e "s3_access_key=xxx s3_secret_key=yyy" - success_msg: "S3 credentials validated" - - - name: Display S3 configuration - ansible.builtin.debug: - msg: | - ======================================== - S3FS Setup on {{ inventory_hostname }} - ======================================== - Target: {{ inventory_hostname }} - S3 Region: {{ s3_region }} - S3 Endpoint: {{ s3_endpoint }} - - Buckets to mount: - {% for bucket in s3_buckets %} - - {{ bucket.name }} -> {{ bucket.mount_point }} - {% endfor %} - ======================================== - - roles: - - role: s3fs - - post_tasks: - - name: Display mounted filesystems - ansible.builtin.command: - cmd: df -h - register: df_output - changed_when: false - - - name: Show S3 mounts - ansible.builtin.debug: - msg: "{{ df_output.stdout_lines | select('search', 's3fs') | list }}" - - - name: Create immich library directory structure - ansible.builtin.file: - path: "{{ item }}" - state: directory - mode: '0775' - owner: root - group: docker - loop: - - /mnt/s3/immich/library - - /mnt/s3/immich/library/backups - - /mnt/s3/immich/library/encoded-video - - /mnt/s3/immich/library/library - - /mnt/s3/immich/library/profile - - /mnt/s3/immich/library/thumbs - - /mnt/s3/immich/library/upload - when: inventory_hostname in groups['services_servers'] - - - name: Display post-setup information - ansible.builtin.debug: - msg: | - ======================================== - S3FS Setup Complete on {{ inventory_hostname }}! - ======================================== - - ✅ All S3 buckets mounted successfully - - Mount points: - {% for bucket in s3_buckets %} - - {{ bucket.mount_point }} ({{ bucket.name }}) - {% endfor %} - - Cache configuration: - - Location: /var/cache/s3fs - - Max size: ~{{ (40 - (s3fs_ensure_diskfree / 1024)) | int }}GB - - Free space reserved: {{ (s3fs_ensure_diskfree / 1024) | int }}GB - - {% if inventory_hostname in groups['services_servers'] %} - Immich library structure created at: - /mnt/s3/immich/library/ - ├── backups/ - ├── encoded-video/ - ├── library/ - ├── profile/ - ├── thumbs/ - └── upload/ - - Next steps for Immich migration: - 1. Manually sync data from legacy server: - rsync -avz --info=progress2 \ - asxpio-old:/srv/immich/library/ \ - /mnt/s3/immich/library/ - - 2. Backup Immich PostgreSQL database on legacy server: - docker exec immich_postgres pg_dumpall -U postgres | gzip > /tmp/immich-db.sql.gz - - 3. Deploy Immich service configured to use S3 storage: - IMMICH_MEDIA_LOCATION=/mnt/s3/immich/library - {% endif %} - - ======================================== diff --git a/ansible/roles/forgejo_runner/handlers/main.yaml b/ansible/roles/forgejo_runner/handlers/main.yaml deleted file mode 100644 index 4ece759..0000000 --- a/ansible/roles/forgejo_runner/handlers/main.yaml +++ /dev/null @@ -1,5 +0,0 @@ ---- -- name: Restart Forgejo Runner - community.docker.docker_compose_v2: - project_src: /opt/forgejo-runner - state: restarted diff --git a/ansible/roles/forgejo_runner/tasks/main.yaml b/ansible/roles/forgejo_runner/tasks/main.yaml deleted file mode 100644 index 0269082..0000000 --- a/ansible/roles/forgejo_runner/tasks/main.yaml +++ /dev/null @@ -1,74 +0,0 @@ ---- -- name: Validate Forgejo Runner variables - ansible.builtin.import_tasks: validate.yaml - -- name: Deploy Forgejo Runner using shared docker_service role - ansible.builtin.include_role: - name: docker_service - vars: - service_name: "forgejo-runner" - service_base_path: "{{ forgejo_runner_base_path }}" - service_directories: - - { path: "data" } - - { path: "config" } - service_templates: - - { src: "config.yaml.j2", dest: "config/config.yaml" } - - { src: "docker-compose.yml.j2", dest: "docker-compose.yml" } - health_check_enabled: false # the runner exposes no port - -- name: Check if runner is already registered - ansible.builtin.stat: - path: "{{ forgejo_runner_base_path }}/data/.runner" - register: runner_registered - -- name: Set permissions on data directory - ansible.builtin.file: - path: "{{ forgejo_runner_base_path }}/data" - state: directory - mode: '0777' - recurse: true - when: not runner_registered.stat.exists - -- name: Stop runner for registration - community.docker.docker_compose_v2: - project_src: "{{ forgejo_runner_base_path }}" - state: stopped - when: not runner_registered.stat.exists - -- name: Register runner with Forgejo - ansible.builtin.command: - cmd: > - docker run --rm - -v {{ forgejo_runner_base_path }}/data:/data - -v {{ forgejo_runner_base_path }}/config/config.yaml:/config.yaml - code.forgejo.org/forgejo/runner:{{ forgejo_runner_version }} - forgejo-runner register - --no-interactive - --token {{ forgejo_runner_token }} - --name runner-{{ inventory_hostname }} - --instance http://{{ asxpio_ops_vpc_ip }}:{{ forgejo_http_port }} - --labels ubuntu-latest:docker://node:20-bookworm,ubuntu-22.04:docker://node:20-bookworm,ubuntu-24.04:docker://node:22-bookworm,debian-latest:docker://debian:bookworm,debian-12:docker://debian:bookworm,debian-11:docker://debian:bullseye,arch-latest:docker://archlinux:latest,nginx-latest:docker://nginx:latest,nginx-alpine:docker://nginx:alpine,node-20:docker://node:20,node-22:docker://node:22,python-3.11:docker://python:3.11,python-3.12:docker://python:3.12,alpine-latest:docker://alpine:latest - when: not runner_registered.stat.exists - changed_when: true - register: registration_result - -- name: Start runner after registration - community.docker.docker_compose_v2: - project_src: "{{ forgejo_runner_base_path }}" - state: present - when: not runner_registered.stat.exists - -- name: Display runner status - ansible.builtin.debug: - msg: | - Forgejo Runner deployed successfully! - - Runner name: runner-{{ inventory_hostname }} - Forgejo instance: http://{{ asxpio_ops_vpc_ip }}:{{ forgejo_http_port }} - {% if runner_registered.stat.exists %} - Status: Already registered (using existing registration) - {% else %} - Status: Newly registered - {% endif %} - - Check runner status: docker logs forgejo-runner diff --git a/ansible/roles/forgejo_runner/tasks/validate.yaml b/ansible/roles/forgejo_runner/tasks/validate.yaml deleted file mode 100644 index a0a2987..0000000 --- a/ansible/roles/forgejo_runner/tasks/validate.yaml +++ /dev/null @@ -1,26 +0,0 @@ ---- -# Variable validation for forgejo_runner role - -- name: Validate Forgejo Runner required variables - ansible.builtin.assert: - that: - - forgejo_runner_base_path is defined - - forgejo_runner_base_path | length > 0 - - forgejo_runner_token is defined - - forgejo_runner_token | length > 0 - - forgejo_runner_version is defined - - forgejo_runner_version | length > 0 - - forgejo_http_port is defined - - forgejo_http_port | int > 0 - - asxpio_ops_vpc_ip is defined - - asxpio_ops_vpc_ip | length > 0 - fail_msg: | - Forgejo Runner role requires: - - forgejo_runner_base_path (installation directory) - - forgejo_runner_token (runner registration token) - - forgejo_runner_version (e.g., '3.5.1') - - forgejo_http_port (Forgejo HTTP port) - - asxpio_ops_vpc_ip (Forgejo server IP) - - WARNING: forgejo_runner_token should be stored in Ansible Vault! - success_msg: "Forgejo Runner variables validated (WARNING: token should use Ansible Vault)" diff --git a/ansible/roles/forgejo_runner/templates/config.yaml.j2 b/ansible/roles/forgejo_runner/templates/config.yaml.j2 deleted file mode 100644 index b24ad71..0000000 --- a/ansible/roles/forgejo_runner/templates/config.yaml.j2 +++ /dev/null @@ -1,48 +0,0 @@ -log: - level: info - job_level: info - -runner: - file: /data/.runner - capacity: {{ forgejo_runner_capacity }} - envs: {} - env_file: .env - timeout: {{ forgejo_runner_timeout }} - shutdown_timeout: {{ forgejo_runner_timeout }} - insecure: false - fetch_timeout: 5s - fetch_interval: 2s - report_interval: 1s - labels: - - "ubuntu-latest:docker://node:20-bookworm" - - "ubuntu-22.04:docker://node:20-bookworm" - - "ubuntu-24.04:docker://node:22-bookworm" - - "debian-latest:docker://debian:bookworm" - - "debian-12:docker://debian:bookworm" - - "debian-11:docker://debian:bullseye" - - "arch-latest:docker://archlinux:latest" - - "nginx-latest:docker://nginx:latest" - - "nginx-alpine:docker://nginx:alpine" - - "node-20:docker://node:20" - - "node-22:docker://node:22" - - "python-3.11:docker://python:3.11" - - "python-3.12:docker://python:3.12" - - "alpine-latest:docker://alpine:latest" - - "nix-latest:docker://nixos/nix:{{ forgejo_runner_nix_version }}" - -cache: - enabled: true - port: 0 - dir: "" - -container: - # "" so each workflow gets a bridge network; services (Postgres) need it. - network: "" - enable_ipv6: false - privileged: false - options: "-v nix-store-{{ forgejo_runner_nix_version }}:/nix" - valid_volumes: - - '**' - docker_host: "automount" - force_pull: false - force_rebuild: false diff --git a/ansible/roles/forgejo_runner/templates/docker-compose.yml.j2 b/ansible/roles/forgejo_runner/templates/docker-compose.yml.j2 deleted file mode 100644 index 1f9bd17..0000000 --- a/ansible/roles/forgejo_runner/templates/docker-compose.yml.j2 +++ /dev/null @@ -1,16 +0,0 @@ -name: forgejo-runner - -services: - forgejo-runner: - container_name: forgejo-runner - image: code.forgejo.org/forgejo/runner:{{ forgejo_runner_version }} - restart: unless-stopped - user: "0:0" # Run as root to access Docker socket - command: forgejo-runner daemon --config /config.yaml - environment: - - CONFIG_FILE=/config.yaml - volumes: - - ./data:/data - - ./config/config.yaml:/config.yaml - - /var/run/docker.sock:/var/run/docker.sock - network_mode: host diff --git a/ansible/roles/minecraft/files/docker-compose.yml b/ansible/roles/minecraft/files/docker-compose.yml deleted file mode 100644 index 5251913..0000000 --- a/ansible/roles/minecraft/files/docker-compose.yml +++ /dev/null @@ -1,15 +0,0 @@ -services: - minecraft: - image: eclipse-temurin:21-jre - container_name: minecraft - working_dir: /data - command: java -Xmx4G -Xms2G -jar paper-1.21.11-54.jar nogui - stdin_open: true - tty: true - restart: unless-stopped - stop_grace_period: 2m - ports: - - "25565:25565" - - "19132:19132/udp" - volumes: - - /opt/minecraft:/data diff --git a/ansible/roles/minecraft/tasks/main.yaml b/ansible/roles/minecraft/tasks/main.yaml deleted file mode 100644 index 6656822..0000000 --- a/ansible/roles/minecraft/tasks/main.yaml +++ /dev/null @@ -1,15 +0,0 @@ ---- -# Paper server; the world and the server jar live next to the compose -# file in /opt/minecraft and are not managed here. A compose change -# recreates the container, so it is stopped with time to save first. -- name: Deploy Minecraft - ansible.builtin.include_role: - name: docker_service - vars: - service_name: "minecraft" - service_base_path: "{{ services_base_path }}/minecraft" - service_files: - - { src: "docker-compose.yml", dest: "docker-compose.yml" } - service_graceful_stop: - - { container: minecraft, timeout: 120 } - health_check_enabled: false diff --git a/ansible/roles/minecraft_fossils/files/docker-compose.yml b/ansible/roles/minecraft_fossils/files/docker-compose.yml deleted file mode 100644 index c937a27..0000000 --- a/ansible/roles/minecraft_fossils/files/docker-compose.yml +++ /dev/null @@ -1,87 +0,0 @@ -services: - minecraft-fossils: - image: itzg/minecraft-server:java21 - container_name: minecraft-fossils - restart: unless-stopped - stop_grace_period: 2m - stdin_open: true - tty: true - ports: - - "25566:25565" - env_file: - - secrets.env - environment: - EULA: "TRUE" - TYPE: FABRIC - VERSION: "1.20.1" - FABRIC_LOADER_VERSION: "0.18.4" - MEMORY: "8G" - ONLINE_MODE: "FALSE" - ENFORCE_WHITELIST: "TRUE" - ENFORCE_SECURE_PROFILE: "FALSE" - WHITELIST_FILE: /data/whitelist.json - OPS_FILE: /data/ops.json - ENABLE_RCON: "TRUE" - MOTD: "[mc.czsk.dev] Fossils Archaeology Revival: Genesis" - USE_AIKAR_FLAGS: "TRUE" - volumes: - - /opt/minecraft-fossils/data:/data - networks: - - fossils - - whitelist-sync: - image: alpine:3 - container_name: minecraft-fossils-wlsync - restart: unless-stopped - depends_on: - - minecraft-fossils - volumes: - - /opt/minecraft:/src:ro - - /opt/minecraft-fossils/data:/dst - env_file: - - secrets.env - environment: - RCON_HOST: minecraft-fossils - INTERVAL: "15" - command: - - sh - - -c - - | - apk add --no-cache python3 >/dev/null - cat > /rcon.py <<'PY' - import socket, struct, sys, os - host, port, pw, cmd = os.environ["RCON_HOST"], 25575, os.environ["RCON_PASSWORD"], sys.argv[1] - def pkt(i, t, b): p = struct.pack("/dev/null; then - cp /src/$$f /dst/$$f - echo "[wlsync] copied $$f" - case $$f in - whitelist.json) CMD="whitelist reload" ;; - ops.json) CMD="reload" ;; - banned-players.json|banned-ips.json) CMD="kick @a Ban list updated" ;; - esac - python3 /rcon.py "$$CMD" 2>&1 | sed 's/^/[wlsync] rcon: /' || echo "[wlsync] rcon failed" - fi - done - sleep $$INTERVAL - done - networks: - - fossils - -networks: - fossils: - driver: bridge diff --git a/ansible/roles/minecraft_fossils/files/minecraft-fossils-restart.service b/ansible/roles/minecraft_fossils/files/minecraft-fossils-restart.service deleted file mode 100644 index 7798cd6..0000000 --- a/ansible/roles/minecraft_fossils/files/minecraft-fossils-restart.service +++ /dev/null @@ -1,8 +0,0 @@ -[Unit] -Description=Daily restart of minecraft-fossils container -Requires=docker.service -After=docker.service - -[Service] -Type=oneshot -ExecStart=/usr/bin/docker restart minecraft-fossils diff --git a/ansible/roles/minecraft_fossils/files/minecraft-fossils-restart.timer b/ansible/roles/minecraft_fossils/files/minecraft-fossils-restart.timer deleted file mode 100644 index aee390a..0000000 --- a/ansible/roles/minecraft_fossils/files/minecraft-fossils-restart.timer +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=Restart minecraft-fossils daily at 08:00 Europe/Tallinn - -[Timer] -OnCalendar=*-*-* 08:00:00 Europe/Tallinn -Persistent=true -Unit=minecraft-fossils-restart.service - -[Install] -WantedBy=timers.target diff --git a/ansible/roles/minecraft_fossils/handlers/main.yaml b/ansible/roles/minecraft_fossils/handlers/main.yaml deleted file mode 100644 index c28484f..0000000 --- a/ansible/roles/minecraft_fossils/handlers/main.yaml +++ /dev/null @@ -1,4 +0,0 @@ ---- -- name: Reload systemd - ansible.builtin.systemd: - daemon_reload: true diff --git a/ansible/roles/minecraft_fossils/tasks/main.yaml b/ansible/roles/minecraft_fossils/tasks/main.yaml deleted file mode 100644 index 1e20de5..0000000 --- a/ansible/roles/minecraft_fossils/tasks/main.yaml +++ /dev/null @@ -1,49 +0,0 @@ ---- -# Fabric server plus a sidecar that mirrors the whitelist, ops and ban -# lists from the Paper server. World data in /opt/minecraft-fossils/data -# is not managed here. A compose change recreates the container, so it -# is saved and stopped with time to finish first. -- name: Deploy Minecraft Fossils - ansible.builtin.include_role: - name: docker_service - vars: - service_name: "minecraft-fossils" - service_base_path: "{{ services_base_path }}/minecraft-fossils" - service_templates: - - src: "secrets.env.j2" - dest: "secrets.env" - owner: "{{ minecraft_fossils_owner }}" - group: "{{ minecraft_fossils_owner }}" - mode: "0600" - service_files: - - src: "docker-compose.yml" - dest: "docker-compose.yml" - owner: "{{ minecraft_fossils_owner }}" - group: "{{ minecraft_fossils_owner }}" - # secrets.env only matters at container start; never restart the - # game server as a side effect of a deploy - service_restart_on_change: false - service_graceful_stop: - - container: minecraft-fossils - timeout: 120 - pre_stop: docker exec minecraft-fossils rcon-cli save-all flush - health_check_enabled: false - -- name: Install the daily restart units - ansible.builtin.copy: - src: "{{ item }}" - dest: "/etc/systemd/system/{{ item }}" - mode: "0644" - loop: - - minecraft-fossils-restart.service - - minecraft-fossils-restart.timer - notify: Reload systemd - -- name: Flush handlers - ansible.builtin.meta: flush_handlers - -- name: Enable the daily restart timer - ansible.builtin.systemd: - name: minecraft-fossils-restart.timer - enabled: true - state: started diff --git a/ansible/roles/minecraft_fossils/templates/secrets.env.j2 b/ansible/roles/minecraft_fossils/templates/secrets.env.j2 deleted file mode 100644 index 616eec2..0000000 --- a/ansible/roles/minecraft_fossils/templates/secrets.env.j2 +++ /dev/null @@ -1,2 +0,0 @@ -RCON_PASSWORD={{ minecraft_fossils_rcon_password }} -CF_API_KEY={{ minecraft_fossils_cf_api_key }} diff --git a/ansible/roles/s3fs/defaults/main.yaml b/ansible/roles/s3fs/defaults/main.yaml deleted file mode 100644 index dbcdfd1..0000000 --- a/ansible/roles/s3fs/defaults/main.yaml +++ /dev/null @@ -1,56 +0,0 @@ ---- -# S3FS role defaults - -# S3 Configuration -s3_endpoint: "https://{{ s3_region }}.your-objectstorage.com" -s3_region: "hel1" - -# S3 Credentials (should be overridden in group_vars or vault) -s3_access_key: "" -s3_secret_key: "" - -# false: no mounts. Unmounts and drops the fstab entries; the credentials -# file stays (backups use it with rclone, /root/backups/rclone-hetzner.sh). -s3fs_enabled: true - -# S3 Buckets to mount (non-replication buckets only) -s3_buckets: - - name: asxpio-terraform-main - mount_point: /mnt/s3/terraform - owner: root - group: root - mode: "0755" - - name: asxpio-backups-main - mount_point: /mnt/s3/backups - owner: root - group: root - mode: "0755" - - name: asxpio-immich-main - mount_point: /mnt/s3/immich - owner: root - group: docker - mode: "0775" - -# S3FS mount options -# Note: s3fs_ensure_diskfree is defined per-host in host_vars/ -s3fs_options: - - "passwd_file=/etc/s3fs/.passwd-s3fs" - - "use_path_request_style" - - "url={{ s3_endpoint }}" - - "allow_other" - - "umask=0022" - - "uid={{ s3fs_uid | default('0') }}" - - "gid={{ s3fs_gid | default('0') }}" - - "use_cache=/var/cache/s3fs" - - "ensure_diskfree={{ s3fs_ensure_diskfree | default(81920) }}" - - "max_stat_cache_size=10000" - - "stat_cache_expire=900" - - "parallel_count=15" - - "multipart_size=52" - - "nocopyapi" - - "enable_noobj_cache" - -# Package dependencies -s3fs_packages: - - s3fs - - fuse3 diff --git a/ansible/roles/s3fs/tasks/main.yaml b/ansible/roles/s3fs/tasks/main.yaml deleted file mode 100644 index ac1aa66..0000000 --- a/ansible/roles/s3fs/tasks/main.yaml +++ /dev/null @@ -1,120 +0,0 @@ ---- -# S3FS role tasks - Mount S3 buckets using s3fs - -- name: Install s3fs and dependencies - ansible.builtin.apt: - name: "{{ s3fs_packages }}" - state: present - update_cache: true - -- name: Create s3fs credentials directory - ansible.builtin.file: - path: /etc/s3fs - state: directory - mode: '0700' - owner: root - group: root - -- name: Create s3fs password file - ansible.builtin.template: - src: passwd-s3fs.j2 - dest: /etc/s3fs/.passwd-s3fs - mode: '0600' - owner: root - group: root - no_log: true - -- name: Create S3 cache directory - ansible.builtin.file: - path: /var/cache/s3fs - state: directory - mode: '0755' - owner: root - group: root - -- name: Mount S3 buckets - when: s3fs_enabled | bool - block: - - name: Create mount points for S3 buckets - ansible.builtin.file: - path: "{{ item.mount_point }}" - state: directory - mode: "{{ item.mode }}" - owner: "{{ item.owner }}" - group: "{{ item.group }}" - loop: "{{ s3_buckets }}" - loop_control: - label: "{{ item.name }} -> {{ item.mount_point }}" - - - name: Add S3 buckets to /etc/fstab - ansible.builtin.lineinfile: - path: /etc/fstab - line: "{{ item.name }} {{ item.mount_point }} fuse.s3fs _netdev,{{ s3fs_options | join(',') }} 0 0" - regexp: "^{{ item.name }}\\s+{{ item.mount_point }}" - state: present - loop: "{{ s3_buckets }}" - loop_control: - label: "{{ item.name }} -> {{ item.mount_point }}" - - - name: Mount all S3 buckets - ansible.builtin.command: - cmd: "mount {{ item.mount_point }}" - loop: "{{ s3_buckets }}" - loop_control: - label: "{{ item.name }} -> {{ item.mount_point }}" - register: mount_result - changed_when: mount_result.rc == 0 - failed_when: false - - - name: Verify S3 buckets are mounted - ansible.builtin.command: - cmd: "mountpoint -q {{ item.mount_point }}" - loop: "{{ s3_buckets }}" - loop_control: - label: "{{ item.name }} -> {{ item.mount_point }}" - register: verify_mount - changed_when: false - failed_when: verify_mount.rc != 0 - - - name: Display mounted S3 buckets - ansible.builtin.debug: - msg: | - ✅ S3 Buckets mounted successfully: - {% for bucket in s3_buckets %} - - {{ bucket.name }} -> {{ bucket.mount_point }} - {% endfor %} - - - name: Test write access to Immich bucket - ansible.builtin.copy: - content: "S3FS test file created on {{ ansible_date_time.iso8601 }}" - dest: "{{ s3_buckets | selectattr('name', 'equalto', 'asxpio-immich-main') | map(attribute='mount_point') | first }}/.s3fs-test" - mode: '0644' - when: "'asxpio-immich-main' in s3_buckets | map(attribute='name')" - register: write_test - - - name: Remove test file - ansible.builtin.file: - path: "{{ s3_buckets | selectattr('name', 'equalto', 'asxpio-immich-main') | map(attribute='mount_point') | first }}/.s3fs-test" - state: absent - when: write_test is changed - -- name: Unmount S3 buckets - ansible.builtin.command: - cmd: "umount {{ item.mount_point }}" - loop: "{{ s3_buckets }}" - loop_control: - label: "{{ item.mount_point }}" - register: umount_result - changed_when: umount_result.rc == 0 - failed_when: false - when: not (s3fs_enabled | bool) - -- name: Remove S3 buckets from /etc/fstab - ansible.builtin.lineinfile: - path: /etc/fstab - regexp: "^{{ item.name }}\\s+{{ item.mount_point }}\\s" - state: absent - loop: "{{ s3_buckets }}" - loop_control: - label: "{{ item.name }} -> {{ item.mount_point }}" - when: not (s3fs_enabled | bool) diff --git a/ansible/roles/s3fs/templates/passwd-s3fs.j2 b/ansible/roles/s3fs/templates/passwd-s3fs.j2 deleted file mode 100644 index 11f3ec7..0000000 --- a/ansible/roles/s3fs/templates/passwd-s3fs.j2 +++ /dev/null @@ -1 +0,0 @@ -{{ s3_access_key }}:{{ s3_secret_key }} diff --git a/ansible/roles/sillytavern/files/docker-compose.yml b/ansible/roles/sillytavern/files/docker-compose.yml deleted file mode 100644 index b2453d6..0000000 --- a/ansible/roles/sillytavern/files/docker-compose.yml +++ /dev/null @@ -1,11 +0,0 @@ -services: - sillytavern: - image: ghcr.io/sillytavern/sillytavern:1.19.0 - container_name: sillytavern - restart: unless-stopped - ports: - - "10.20.0.20:8000:8000" - volumes: - - ./config:/home/node/app/config - - ./data:/home/node/app/data - - ./plugins:/home/node/app/plugins diff --git a/ansible/roles/sillytavern/tasks/main.yaml b/ansible/roles/sillytavern/tasks/main.yaml deleted file mode 100644 index b616e77..0000000 --- a/ansible/roles/sillytavern/tasks/main.yaml +++ /dev/null @@ -1,16 +0,0 @@ ---- -# Reachable on the WireGuard address only. Bump the image tag in -# files/docker-compose.yml to upgrade. -- name: Deploy SillyTavern - ansible.builtin.include_role: - name: docker_service - vars: - service_name: "sillytavern" - service_base_path: "{{ services_base_path }}/sillytavern" - service_directories: - - { path: "config" } - - { path: "data" } - - { path: "plugins" } - service_files: - - { src: "docker-compose.yml", dest: "docker-compose.yml" } - health_check_enabled: false diff --git a/ansible/site.yaml b/ansible/site.yaml index f1a81ca..aaa918e 100644 --- a/ansible/site.yaml +++ b/ansible/site.yaml @@ -1,15 +1,16 @@ --- -# Everything the deploy pipeline manages. Idempotent: a run with nothing +# Everything the deploy pipeline manages on asxpio-ops. asxpio-services +# runs NixOS and is described in ../nixos. Idempotent: a run with nothing # to change touches nothing and restarts nothing. # # ansible-playbook site.yaml --check --diff # dry run # ansible-playbook site.yaml --tags traefik # one service # -# Host bootstrap (WireGuard, SSH hardening, s3fs, swap) is in playbooks/ +# Host bootstrap (WireGuard, SSH hardening, swap) is in playbooks/ # and is run by hand. - name: Host baseline - hosts: asxpio + hosts: ops_servers become: true roles: - { role: base, tags: [base] } @@ -21,26 +22,9 @@ - { role: traefik, tags: [traefik] } - { role: crawler_alert, tags: [crawler_alert] } -- name: asxpio-services services - hosts: services_servers - become: true - roles: - - { role: sillytavern, tags: [sillytavern] } - - { role: minecraft, tags: [minecraft] } - - { role: minecraft_fossils, tags: [minecraft_fossils] } - # Late: restarting Forgejo briefly cuts the runner off from its job log - name: Forgejo hosts: ops_servers become: true roles: - { role: forgejo, tags: [forgejo] } - -# Last: when the pipeline changes the runner it recreates the container -# that is running the pipeline. Everything else is done by then; that job -# ends as failed and the next run confirms the result. -- name: Forgejo runner - hosts: services_servers - become: true - roles: - - { role: forgejo_runner, tags: [forgejo_runner] }