ansible: daily replica of the hel1 buckets into fsn1

restic writes both regions itself, but the ad-hoc backups under
asxpio-ops/, the Terraform state and the k8s state only ever went to
hel1. Sync each hel1 bucket into its fsn1 twin every morning from
asxpio-services, with rclone pinned like everything else.

restic/ is excluded, so the independent fsn1 restic repositories are
never touched. A run that would delete more than 1000 objects stops and
mails, so a mistake on the hel1 side cannot empty the replica the next
morning. The first dry run copied 23 objects and deleted none.

Assisted-by: Claude:opus-5-5

ansible/roles/s3_replicate/templates/env.j2

@@ -0,0 +1,13 @@
+RCLONE_CONFIG_HEL1_TYPE=s3
+RCLONE_CONFIG_HEL1_PROVIDER=Other
+RCLONE_CONFIG_HEL1_ENDPOINT=https://hel1.your-objectstorage.com
+RCLONE_CONFIG_HEL1_REGION=hel1
+RCLONE_CONFIG_HEL1_ACCESS_KEY_ID={{ s3_access_key }}
+RCLONE_CONFIG_HEL1_SECRET_ACCESS_KEY={{ s3_secret_key }}
+RCLONE_CONFIG_FSN1_TYPE=s3
+RCLONE_CONFIG_FSN1_PROVIDER=Other
+RCLONE_CONFIG_FSN1_ENDPOINT=https://fsn1.your-objectstorage.com
+RCLONE_CONFIG_FSN1_REGION=fsn1
+RCLONE_CONFIG_FSN1_ACCESS_KEY_ID={{ s3_access_key }}
+RCLONE_CONFIG_FSN1_SECRET_ACCESS_KEY={{ s3_secret_key }}
+RCLONE_CONFIG_FSN1_NO_CHECK_BUCKET=true

ansible/roles/s3_replicate/templates/s3-replicate.j2

@@ -0,0 +1,19 @@
+#!/bin/bash
+# Managed by Ansible (role s3_replicate). hel1 -> fsn1, one bucket at a time;
+# a failing pair does not stop the others, but fails the run.
+set -uo pipefail
+failed=0
+rclone() {
+  docker run --rm --env-file /etc/s3-replicate/env \
+    rclone/rclone:{{ s3_replicate_rclone_version }} "$@"
+}
+{% for p in s3_replicate_pairs %}
+echo "s3-replicate: {{ p.src }} -> {{ p.dst }}"
+rclone sync hel1:{{ p.src }} fsn1:{{ p.dst }} \
+  --fast-list --checkers 16 --transfers 8 --max-delete {{ s3_replicate_max_delete }} \
+{% for e in p.exclude | default([]) %}
+  --exclude '{{ e }}' \
+{% endfor %}
+  --stats-one-line -v || failed=1
+{% endfor %}
+exit $failed

ansible/roles/s3_replicate/templates/s3-replicate.service.j2

@@ -0,0 +1,11 @@
+[Unit]
+Description=Replicate the hel1 buckets to fsn1
+Wants=network-online.target
+After=network-online.target docker.service
+# Mailer from the restic_backup role
+OnFailure=restic-alert@%n.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/s3-replicate
+TimeoutStartSec=12h

ansible/roles/s3_replicate/templates/s3-replicate.timer.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=Daily hel1 -> fsn1 bucket replica
+
+[Timer]
+OnCalendar={{ s3_replicate_schedule }}
+RandomizedDelaySec=15m
+Persistent=true
+
+[Install]
+WantedBy=timers.target