diff --git a/ansible/roles/s3_replicate/defaults/main.yaml b/ansible/roles/s3_replicate/defaults/main.yaml new file mode 100644 index 0000000..a818885 --- /dev/null +++ b/ansible/roles/s3_replicate/defaults/main.yaml @@ -0,0 +1,16 @@ +--- +# Daily one-way replica of the hel1 buckets into their fsn1 twins. +# restic/ is excluded: restic writes its own, independent fsn1 repository. + +# renovate: datasource=docker depName=rclone/rclone +s3_replicate_rclone_version: "1.76.0" + +s3_replicate_pairs: + - { src: asxpio-backups-main, dst: asxpio-backups-repl, exclude: ["restic/**"] } + - { src: asxpio-terraform-main, dst: asxpio-terraform-repl } + - { src: k8s-asxpio-main, dst: k8s-asxpio-repl } + +# A run that would delete more than this stops and fails (and mails): +# a mistake on the hel1 side must not empty the replica the next morning +s3_replicate_max_delete: 1000 +s3_replicate_schedule: "*-*-* 06:00:00" diff --git a/ansible/roles/s3_replicate/handlers/main.yaml b/ansible/roles/s3_replicate/handlers/main.yaml new file mode 100644 index 0000000..c28484f --- /dev/null +++ b/ansible/roles/s3_replicate/handlers/main.yaml @@ -0,0 +1,4 @@ +--- +- name: Reload systemd + ansible.builtin.systemd: + daemon_reload: true diff --git a/ansible/roles/s3_replicate/tasks/main.yaml b/ansible/roles/s3_replicate/tasks/main.yaml new file mode 100644 index 0000000..df00c5d --- /dev/null +++ b/ansible/roles/s3_replicate/tasks/main.yaml @@ -0,0 +1,38 @@ +--- +- name: Create the config directory + ansible.builtin.file: + path: /etc/s3-replicate + state: directory + mode: "0700" + +- name: Write credentials + ansible.builtin.template: + src: env.j2 + dest: /etc/s3-replicate/env + mode: "0600" + diff: false + +- name: Install the script + ansible.builtin.template: + src: s3-replicate.j2 + dest: /usr/local/sbin/s3-replicate + mode: "0700" + +- name: Install the systemd units + ansible.builtin.template: + src: "{{ item }}.j2" + dest: "/etc/systemd/system/{{ item }}" + mode: "0644" + loop: [s3-replicate.service, s3-replicate.timer] + notify: Reload systemd + +- name: Flush handlers + ansible.builtin.meta: flush_handlers + +- name: Enable the daily timer + ansible.builtin.systemd: + name: s3-replicate.timer + enabled: true + state: started + # On a first dry run the unit is not installed yet + ignore_errors: "{{ ansible_check_mode }}" diff --git a/ansible/roles/s3_replicate/templates/env.j2 b/ansible/roles/s3_replicate/templates/env.j2 new file mode 100644 index 0000000..b98e812 --- /dev/null +++ b/ansible/roles/s3_replicate/templates/env.j2 @@ -0,0 +1,13 @@ +RCLONE_CONFIG_HEL1_TYPE=s3 +RCLONE_CONFIG_HEL1_PROVIDER=Other +RCLONE_CONFIG_HEL1_ENDPOINT=https://hel1.your-objectstorage.com +RCLONE_CONFIG_HEL1_REGION=hel1 +RCLONE_CONFIG_HEL1_ACCESS_KEY_ID={{ s3_access_key }} +RCLONE_CONFIG_HEL1_SECRET_ACCESS_KEY={{ s3_secret_key }} +RCLONE_CONFIG_FSN1_TYPE=s3 +RCLONE_CONFIG_FSN1_PROVIDER=Other +RCLONE_CONFIG_FSN1_ENDPOINT=https://fsn1.your-objectstorage.com +RCLONE_CONFIG_FSN1_REGION=fsn1 +RCLONE_CONFIG_FSN1_ACCESS_KEY_ID={{ s3_access_key }} +RCLONE_CONFIG_FSN1_SECRET_ACCESS_KEY={{ s3_secret_key }} +RCLONE_CONFIG_FSN1_NO_CHECK_BUCKET=true diff --git a/ansible/roles/s3_replicate/templates/s3-replicate.j2 b/ansible/roles/s3_replicate/templates/s3-replicate.j2 new file mode 100644 index 0000000..88f2741 --- /dev/null +++ b/ansible/roles/s3_replicate/templates/s3-replicate.j2 @@ -0,0 +1,19 @@ +#!/bin/bash +# Managed by Ansible (role s3_replicate). hel1 -> fsn1, one bucket at a time; +# a failing pair does not stop the others, but fails the run. +set -uo pipefail +failed=0 +rclone() { + docker run --rm --env-file /etc/s3-replicate/env \ + rclone/rclone:{{ s3_replicate_rclone_version }} "$@" +} +{% for p in s3_replicate_pairs %} +echo "s3-replicate: {{ p.src }} -> {{ p.dst }}" +rclone sync hel1:{{ p.src }} fsn1:{{ p.dst }} \ + --fast-list --checkers 16 --transfers 8 --max-delete {{ s3_replicate_max_delete }} \ +{% for e in p.exclude | default([]) %} + --exclude '{{ e }}' \ +{% endfor %} + --stats-one-line -v || failed=1 +{% endfor %} +exit $failed diff --git a/ansible/roles/s3_replicate/templates/s3-replicate.service.j2 b/ansible/roles/s3_replicate/templates/s3-replicate.service.j2 new file mode 100644 index 0000000..96d63a8 --- /dev/null +++ b/ansible/roles/s3_replicate/templates/s3-replicate.service.j2 @@ -0,0 +1,11 @@ +[Unit] +Description=Replicate the hel1 buckets to fsn1 +Wants=network-online.target +After=network-online.target docker.service +# Mailer from the restic_backup role +OnFailure=restic-alert@%n.service + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/s3-replicate +TimeoutStartSec=12h diff --git a/ansible/roles/s3_replicate/templates/s3-replicate.timer.j2 b/ansible/roles/s3_replicate/templates/s3-replicate.timer.j2 new file mode 100644 index 0000000..caedb38 --- /dev/null +++ b/ansible/roles/s3_replicate/templates/s3-replicate.timer.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Daily hel1 -> fsn1 bucket replica + +[Timer] +OnCalendar={{ s3_replicate_schedule }} +RandomizedDelaySec=15m +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/site.yaml b/ansible/site.yaml index 9cb1e44..333f253 100644 --- a/ansible/site.yaml +++ b/ansible/site.yaml @@ -44,6 +44,12 @@ roles: - { role: restic_backup, tags: [restic] } +- name: Bucket replica + hosts: services_servers + become: true + roles: + - { role: s3_replicate, tags: [s3_replicate] } + # Last: when the pipeline changes the runner it recreates the container # that is running the pipeline. Everything else is done by then; that job # ends as failed and the next run confirms the result.