ansible: nightly restic backups to both object storage regions

Hetzner's server backups cost 20% of each server's price and only
restore whole disks. Back up what matters with restic instead, into the
backup buckets we already pay for: one repository per host in hel1
(asxpio-backups-main) and one in fsn1 (asxpio-backups-repl), written
independently so either region alone is a full copy.

Postgres goes in as a dump, Forgejo's and vhpbot's SQLite databases
through the online backup API, Garage with a fresh metadata snapshot,
ejabberd through its own backup command. Keeps 7 daily, 4 weekly and
6 monthly snapshots, prunes on Sundays, and mails the journal when a
run fails.

restic is the pinned upstream release, so its version is tracked in
the repo like everything else. The vault gains the restic password
and the Xray config for the role that follows.

Assisted-by: Claude:opus-5-5

ansible/roles/restic_backup/templates/restic-backup.j2

@@ -0,0 +1,70 @@
+#!/bin/bash
+# Managed by Ansible (role restic_backup).
+#
+#   restic-backup run                 hooks, then backup + forget to every repository
+#   restic-backup init                create the repositories that do not exist yet
+#   restic-backup <repo> <args...>    plain restic against one repository, e.g.
+#                                     restic-backup hel1 snapshots
+#                                     restic-backup fsn1 restore latest --target /tmp/r
+#
+# Repositories: {{ restic_repositories.keys() | join(', ') }}
+set -uo pipefail
+
+set -a; . /etc/restic/env; set +a
+export RESTIC_CACHE_DIR=/var/cache/restic
+STAGE={{ restic_stage_dir }}
+
+declare -A REPOS=(
+{% for name, url in restic_repositories.items() %}
+  [{{ name }}]="{{ url }}"
+{% endfor %}
+)
+
+log() { echo "restic-backup: $*"; }
+
+case "${1:-}" in
+init)
+  for name in "${!REPOS[@]}"; do
+    if RESTIC_REPOSITORY="${REPOS[$name]}" restic cat config >/dev/null 2>&1; then
+      log "$name: exists"
+    else
+      RESTIC_REPOSITORY="${REPOS[$name]}" restic init || exit 1
+      log "$name: created"
+    fi
+  done
+  ;;
+
+run)
+  failed=0
+  rm -rf "$STAGE"; install -d -m 700 "$STAGE"
+  export STAGE
+{% for hook in restic_pre_hooks %}
+  log "hook {{ loop.index }}/{{ restic_pre_hooks | length }}"
+  ( set -e; {{ hook }} ) || { log "hook failed"; failed=1; }
+{% endfor %}
+
+  for name in "${!REPOS[@]}"; do
+    export RESTIC_REPOSITORY="${REPOS[$name]}"
+    log "$name: backup"
+    restic backup --host {{ inventory_hostname }} --one-file-system --exclude-caches \
+{% for e in restic_excludes %}
+      --exclude '{{ e }}' \
+{% endfor %}
+      {{ (restic_paths + [restic_stage_dir]) | join(' ') }} || { failed=1; continue; }
+
+    prune=""
+    [ "$(date +%a)" = "{{ restic_prune_weekday }}" ] && prune="--prune"
+    log "$name: forget $prune"
+    restic forget --host {{ inventory_hostname }} {{ restic_keep }} $prune || failed=1
+  done
+
+  rm -rf "$STAGE"
+  exit $failed
+  ;;
+
+*)
+  [ -n "${1:-}" ] && [ -n "${REPOS[$1]:-}" ] || { sed -n '4,10p' "$0"; exit 2; }
+  export RESTIC_REPOSITORY="${REPOS[$1]}"; shift
+  exec restic "$@"
+  ;;
+esac