ansible: daily replica of the hel1 buckets into fsn1

restic writes both regions itself, but the ad-hoc backups under
asxpio-ops/, the Terraform state and the k8s state only ever went to
hel1. Sync each hel1 bucket into its fsn1 twin every morning from
asxpio-services, with rclone pinned like everything else.

restic/ is excluded, so the independent fsn1 restic repositories are
never touched. A run that would delete more than 1000 objects stops and
mails, so a mistake on the hel1 side cannot empty the replica the next
morning. The first dry run copied 23 objects and deleted none.

Assisted-by: Claude:opus-5-5

ansible/roles/s3_replicate/defaults/main.yaml

@@ -0,0 +1,16 @@
+---
+# Daily one-way replica of the hel1 buckets into their fsn1 twins.
+# restic/ is excluded: restic writes its own, independent fsn1 repository.
+
+# renovate: datasource=docker depName=rclone/rclone
+s3_replicate_rclone_version: "1.76.0"
+
+s3_replicate_pairs:
+  - { src: asxpio-backups-main, dst: asxpio-backups-repl, exclude: ["restic/**"] }
+  - { src: asxpio-terraform-main, dst: asxpio-terraform-repl }
+  - { src: k8s-asxpio-main, dst: k8s-asxpio-repl }
+
+# A run that would delete more than this stops and fails (and mails):
+# a mistake on the hel1 side must not empty the replica the next morning
+s3_replicate_max_delete: 1000
+s3_replicate_schedule: "*-*-* 06:00:00"

ansible/roles/s3_replicate/handlers/main.yaml

@@ -0,0 +1,4 @@
+---
+- name: Reload systemd
+  ansible.builtin.systemd:
+    daemon_reload: true

ansible/roles/s3_replicate/tasks/main.yaml

@@ -0,0 +1,38 @@
+---
+- name: Create the config directory
+  ansible.builtin.file:
+    path: /etc/s3-replicate
+    state: directory
+    mode: "0700"
+
+- name: Write credentials
+  ansible.builtin.template:
+    src: env.j2
+    dest: /etc/s3-replicate/env
+    mode: "0600"
+  diff: false
+
+- name: Install the script
+  ansible.builtin.template:
+    src: s3-replicate.j2
+    dest: /usr/local/sbin/s3-replicate
+    mode: "0700"
+
+- name: Install the systemd units
+  ansible.builtin.template:
+    src: "{{ item }}.j2"
+    dest: "/etc/systemd/system/{{ item }}"
+    mode: "0644"
+  loop: [s3-replicate.service, s3-replicate.timer]
+  notify: Reload systemd
+
+- name: Flush handlers
+  ansible.builtin.meta: flush_handlers
+
+- name: Enable the daily timer
+  ansible.builtin.systemd:
+    name: s3-replicate.timer
+    enabled: true
+    state: started
+  # On a first dry run the unit is not installed yet
+  ignore_errors: "{{ ansible_check_mode }}"

ansible/roles/s3_replicate/templates/env.j2

@@ -0,0 +1,13 @@
+RCLONE_CONFIG_HEL1_TYPE=s3
+RCLONE_CONFIG_HEL1_PROVIDER=Other
+RCLONE_CONFIG_HEL1_ENDPOINT=https://hel1.your-objectstorage.com
+RCLONE_CONFIG_HEL1_REGION=hel1
+RCLONE_CONFIG_HEL1_ACCESS_KEY_ID={{ s3_access_key }}
+RCLONE_CONFIG_HEL1_SECRET_ACCESS_KEY={{ s3_secret_key }}
+RCLONE_CONFIG_FSN1_TYPE=s3
+RCLONE_CONFIG_FSN1_PROVIDER=Other
+RCLONE_CONFIG_FSN1_ENDPOINT=https://fsn1.your-objectstorage.com
+RCLONE_CONFIG_FSN1_REGION=fsn1
+RCLONE_CONFIG_FSN1_ACCESS_KEY_ID={{ s3_access_key }}
+RCLONE_CONFIG_FSN1_SECRET_ACCESS_KEY={{ s3_secret_key }}
+RCLONE_CONFIG_FSN1_NO_CHECK_BUCKET=true

ansible/roles/s3_replicate/templates/s3-replicate.j2

@@ -0,0 +1,19 @@
+#!/bin/bash
+# Managed by Ansible (role s3_replicate). hel1 -> fsn1, one bucket at a time;
+# a failing pair does not stop the others, but fails the run.
+set -uo pipefail
+failed=0
+rclone() {
+  docker run --rm --env-file /etc/s3-replicate/env \
+    rclone/rclone:{{ s3_replicate_rclone_version }} "$@"
+}
+{% for p in s3_replicate_pairs %}
+echo "s3-replicate: {{ p.src }} -> {{ p.dst }}"
+rclone sync hel1:{{ p.src }} fsn1:{{ p.dst }} \
+  --fast-list --checkers 16 --transfers 8 --max-delete {{ s3_replicate_max_delete }} \
+{% for e in p.exclude | default([]) %}
+  --exclude '{{ e }}' \
+{% endfor %}
+  --stats-one-line -v || failed=1
+{% endfor %}
+exit $failed

ansible/roles/s3_replicate/templates/s3-replicate.service.j2

@@ -0,0 +1,11 @@
+[Unit]
+Description=Replicate the hel1 buckets to fsn1
+Wants=network-online.target
+After=network-online.target docker.service
+# Mailer from the restic_backup role
+OnFailure=restic-alert@%n.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/s3-replicate
+TimeoutStartSec=12h

ansible/roles/s3_replicate/templates/s3-replicate.timer.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=Daily hel1 -> fsn1 bucket replica
+
+[Timer]
+OnCalendar={{ s3_replicate_schedule }}
+RandomizedDelaySec=15m
+Persistent=true
+
+[Install]
+WantedBy=timers.target

ansible/site.yaml

@@ -44,6 +44,12 @@
   roles:
     - { role: restic_backup, tags: [restic] }
 
+- name: Bucket replica
+  hosts: services_servers
+  become: true
+  roles:
+    - { role: s3_replicate, tags: [s3_replicate] }
+
 # Last: when the pipeline changes the runner it recreates the container
 # that is running the pipeline. Everything else is done by then; that job
 # ends as failed and the next run confirms the result.