ansible: nightly restic backups to both object storage regions

Hetzner's server backups cost 20% of each server's price and only
restore whole disks. Back up what matters with restic instead, into the
backup buckets we already pay for: one repository per host in hel1
(asxpio-backups-main) and one in fsn1 (asxpio-backups-repl), written
independently so either region alone is a full copy.

Postgres goes in as a dump, Forgejo's and vhpbot's SQLite databases
through the online backup API, Garage with a fresh metadata snapshot,
ejabberd through its own backup command. Keeps 7 daily, 4 weekly and
6 monthly snapshots, prunes on Sundays, and mails the journal when a
run fails.

restic is the pinned upstream release, so its version is tracked in
the repo like everything else. The vault gains the restic password
and the Xray config for the role that follows.

Assisted-by: Claude:opus-5-5

ansible/roles/restic_backup/defaults/main.yaml

@@ -0,0 +1,36 @@
+---
+# Nightly restic backup of the host to Hetzner Object Storage, one
+# repository per region, written independently: losing one region or one
+# repository leaves the other complete.
+
+# renovate: datasource=github-releases depName=restic/restic
+restic_version: "0.19.1"
+
+restic_password: "{{ vault_restic_password }}"
+restic_s3_access_key: "{{ s3_access_key }}"
+restic_s3_secret_key: "{{ s3_secret_key }}"
+
+restic_repositories:
+  hel1: "s3:https://hel1.your-objectstorage.com/{{ s3_bucket_backups_main }}/restic/{{ inventory_hostname }}"
+  fsn1: "s3:https://fsn1.your-objectstorage.com/{{ s3_bucket_backups_repl }}/restic/{{ inventory_hostname }}"
+
+# Dumps and snapshots taken by the hooks land here and are backed up too
+restic_stage_dir: /var/backups/restic-stage
+
+restic_paths: [/etc, /opt]
+restic_excludes: []
+# Shell lines run in order before the backup, with $STAGE set. A failing
+# hook marks the run as failed but the backup still runs.
+restic_pre_hooks: []
+
+restic_keep: "--keep-daily 7 --keep-weekly 4 --keep-monthly 6"
+restic_schedule: "*-*-* 03:30:00"
+# Prune (rewrites pack files, so costs S3 requests) once a week only
+restic_prune_weekday: Sun
+
+# Failure mail, through Forgejo's SMTP account
+restic_alert_smtp_host: smtp.fastmail.com
+restic_alert_smtp_port: 587
+restic_alert_smtp_user: "{{ vault_forgejo_smtp_user }}"
+restic_alert_smtp_password: "{{ vault_forgejo_smtp_password }}"
+restic_alert_recipient: "{{ vault_crawler_recipient }}"

ansible/roles/restic_backup/files/restic-alert

@@ -0,0 +1,20 @@
+#!/usr/bin/python3
+# Mail the tail of a failed unit's journal. Usage: restic-alert <unit>
+import os, smtplib, socket, subprocess, sys
+from email.message import EmailMessage
+
+unit = sys.argv[1]
+env = dict(l.rstrip("\n").split("=", 1) for l in open("/etc/restic/env") if "=" in l)
+log = subprocess.run(["journalctl", "-u", unit, "-n", "60", "--no-pager", "-o", "cat"],
+                     capture_output=True, text=True).stdout
+
+msg = EmailMessage()
+msg["Subject"] = f"[{socket.gethostname()}] {unit} failed"
+msg["From"] = env["ALERT_SMTP_USER"]
+msg["To"] = env["ALERT_RECIPIENT"]
+msg.set_content(log)
+
+with smtplib.SMTP(env["ALERT_SMTP_HOST"], int(env["ALERT_SMTP_PORT"]), timeout=30) as s:
+    s.starttls()
+    s.login(env["ALERT_SMTP_USER"], env["ALERT_SMTP_PASSWORD"])
+    s.send_message(msg)

ansible/roles/restic_backup/files/restic-alert@.service

@@ -0,0 +1,6 @@
+[Unit]
+Description=Mail the failure of %i
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/restic-alert %i

ansible/roles/restic_backup/handlers/main.yaml

@@ -0,0 +1,4 @@
+---
+- name: Reload systemd
+  ansible.builtin.systemd:
+    daemon_reload: true

ansible/roles/restic_backup/tasks/main.yaml

@@ -0,0 +1,88 @@
+---
+- name: Install sqlite3 for consistent copies of live databases
+  ansible.builtin.apt:
+    name: [sqlite3, bzip2]
+    state: present
+
+- name: Check the installed restic version
+  ansible.builtin.command: /usr/local/bin/restic version
+  register: restic_installed
+  changed_when: false
+  failed_when: false
+  check_mode: false
+
+- name: Install restic {{ restic_version }}
+  when: ("restic " ~ restic_version ~ " ") not in (restic_installed.stdout | default(''))
+  block:
+    - name: Download the release
+      ansible.builtin.get_url:
+        url: "https://github.com/restic/restic/releases/download/v{{ restic_version }}/restic_{{ restic_version }}_linux_amd64.bz2"
+        dest: "/tmp/restic_{{ restic_version }}.bz2"
+        checksum: "sha256:https://github.com/restic/restic/releases/download/v{{ restic_version }}/SHA256SUMS"
+        mode: "0600"
+
+    - name: Unpack it
+      ansible.builtin.shell: >
+        bunzip2 -c /tmp/restic_{{ restic_version }}.bz2 > /usr/local/bin/restic.new &&
+        chmod 755 /usr/local/bin/restic.new &&
+        mv /usr/local/bin/restic.new /usr/local/bin/restic &&
+        rm /tmp/restic_{{ restic_version }}.bz2
+      changed_when: true
+
+- name: Create config and cache directories
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    mode: "0700"
+  loop: [/etc/restic, /var/cache/restic]
+
+- name: Write credentials
+  ansible.builtin.template:
+    src: env.j2
+    dest: /etc/restic/env
+    mode: "0600"
+  diff: false
+
+- name: Install the scripts
+  ansible.builtin.template:
+    src: restic-backup.j2
+    dest: /usr/local/sbin/restic-backup
+    mode: "0700"
+
+- name: Install the alert script
+  ansible.builtin.copy:
+    src: restic-alert
+    dest: /usr/local/sbin/restic-alert
+    mode: "0700"
+
+- name: Install the systemd units
+  ansible.builtin.template:
+    src: "{{ item }}.j2"
+    dest: "/etc/systemd/system/{{ item }}"
+    mode: "0644"
+  loop: [restic-backup.service, restic-backup.timer]
+  notify: Reload systemd
+
+- name: Install the alert unit
+  ansible.builtin.copy:
+    src: restic-alert@.service
+    dest: /etc/systemd/system/restic-alert@.service
+    mode: "0644"
+  notify: Reload systemd
+
+- name: Flush handlers
+  ansible.builtin.meta: flush_handlers
+
+- name: Create the repositories
+  ansible.builtin.command: /usr/local/sbin/restic-backup init
+  register: restic_init
+  changed_when: "'created' in restic_init.stdout"
+  when: not ansible_check_mode
+
+- name: Enable the nightly timer
+  ansible.builtin.systemd:
+    name: restic-backup.timer
+    enabled: true
+    state: started
+  # On a first dry run the unit is not installed yet
+  ignore_errors: "{{ ansible_check_mode }}"

ansible/roles/restic_backup/templates/env.j2

@@ -0,0 +1,8 @@
+RESTIC_PASSWORD={{ restic_password }}
+AWS_ACCESS_KEY_ID={{ restic_s3_access_key }}
+AWS_SECRET_ACCESS_KEY={{ restic_s3_secret_key }}
+ALERT_SMTP_HOST={{ restic_alert_smtp_host }}
+ALERT_SMTP_PORT={{ restic_alert_smtp_port }}
+ALERT_SMTP_USER={{ restic_alert_smtp_user }}
+ALERT_SMTP_PASSWORD={{ restic_alert_smtp_password }}
+ALERT_RECIPIENT={{ restic_alert_recipient }}

ansible/roles/restic_backup/templates/restic-backup.j2

@@ -0,0 +1,70 @@
+#!/bin/bash
+# Managed by Ansible (role restic_backup).
+#
+#   restic-backup run                 hooks, then backup + forget to every repository
+#   restic-backup init                create the repositories that do not exist yet
+#   restic-backup <repo> <args...>    plain restic against one repository, e.g.
+#                                     restic-backup hel1 snapshots
+#                                     restic-backup fsn1 restore latest --target /tmp/r
+#
+# Repositories: {{ restic_repositories.keys() | join(', ') }}
+set -uo pipefail
+
+set -a; . /etc/restic/env; set +a
+export RESTIC_CACHE_DIR=/var/cache/restic
+STAGE={{ restic_stage_dir }}
+
+declare -A REPOS=(
+{% for name, url in restic_repositories.items() %}
+  [{{ name }}]="{{ url }}"
+{% endfor %}
+)
+
+log() { echo "restic-backup: $*"; }
+
+case "${1:-}" in
+init)
+  for name in "${!REPOS[@]}"; do
+    if RESTIC_REPOSITORY="${REPOS[$name]}" restic cat config >/dev/null 2>&1; then
+      log "$name: exists"
+    else
+      RESTIC_REPOSITORY="${REPOS[$name]}" restic init || exit 1
+      log "$name: created"
+    fi
+  done
+  ;;
+
+run)
+  failed=0
+  rm -rf "$STAGE"; install -d -m 700 "$STAGE"
+  export STAGE
+{% for hook in restic_pre_hooks %}
+  log "hook {{ loop.index }}/{{ restic_pre_hooks | length }}"
+  ( set -e; {{ hook }} ) || { log "hook failed"; failed=1; }
+{% endfor %}
+
+  for name in "${!REPOS[@]}"; do
+    export RESTIC_REPOSITORY="${REPOS[$name]}"
+    log "$name: backup"
+    restic backup --host {{ inventory_hostname }} --one-file-system --exclude-caches \
+{% for e in restic_excludes %}
+      --exclude '{{ e }}' \
+{% endfor %}
+      {{ (restic_paths + [restic_stage_dir]) | join(' ') }} || { failed=1; continue; }
+
+    prune=""
+    [ "$(date +%a)" = "{{ restic_prune_weekday }}" ] && prune="--prune"
+    log "$name: forget $prune"
+    restic forget --host {{ inventory_hostname }} {{ restic_keep }} $prune || failed=1
+  done
+
+  rm -rf "$STAGE"
+  exit $failed
+  ;;
+
+*)
+  [ -n "${1:-}" ] && [ -n "${REPOS[$1]:-}" ] || { sed -n '4,10p' "$0"; exit 2; }
+  export RESTIC_REPOSITORY="${REPOS[$1]}"; shift
+  exec restic "$@"
+  ;;
+esac

ansible/roles/restic_backup/templates/restic-backup.service.j2

@@ -0,0 +1,12 @@
+[Unit]
+Description=restic backup to Hetzner Object Storage ({{ restic_repositories.keys() | join(', ') }})
+Wants=network-online.target
+After=network-online.target docker.service
+OnFailure=restic-alert@%n.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/restic-backup run
+Nice=10
+IOSchedulingClass=idle
+TimeoutStartSec=6h

ansible/roles/restic_backup/templates/restic-backup.timer.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=Nightly restic backup
+
+[Timer]
+OnCalendar={{ restic_schedule }}
+RandomizedDelaySec=10m
+Persistent=true
+
+[Install]
+WantedBy=timers.target