Sergei Poljanski
e20f5a7d07
All checks were successful
deploy / ansible (push) Successful in 7m23s
Nothing watched the servers: tonight's asxpio crash loop was found by hand. On asxpio-ops, Prometheus (30 days), Loki (14 days) and Grafana at gf.asxp.io (WireGuard only), with blackbox_exporter probing every service from outside and postgres_exporter. On both hosts node_exporter and cAdvisor on the VPC address, and Alloy shipping container logs, the journal and Traefik's access log to Loki over the VPC. Traefik and Forgejo expose their metrics; restic and the bucket replica record their last success for node_exporter. Dashboards: an overview as home, plus hosts, containers, Traefik, probes, Postgres and logs. Alerts by email to me@asxp.io: host or service down, certificate under 14 days, disk or memory nearly full, a container restart-looping, Postgres down, backup or replica stale. Assisted-by: Claude:opus-5-5
77 lines
2.5 KiB
Django/Jinja
77 lines
2.5 KiB
Django/Jinja
#!/bin/bash
|
|
# Managed by Ansible (role restic_backup).
|
|
#
|
|
# restic-backup run hooks, then backup + forget to every repository
|
|
# restic-backup init create the repositories that do not exist yet
|
|
# restic-backup <repo> <args...> plain restic against one repository, e.g.
|
|
# restic-backup hel1 snapshots
|
|
# restic-backup fsn1 restore latest --target /tmp/r
|
|
#
|
|
# Repositories: {{ restic_repositories.keys() | join(', ') }}
|
|
set -uo pipefail
|
|
|
|
set -a; . /etc/restic/env; set +a
|
|
export RESTIC_CACHE_DIR=/var/cache/restic
|
|
STAGE={{ restic_stage_dir }}
|
|
|
|
declare -A REPOS=(
|
|
{% for name, url in restic_repositories.items() %}
|
|
[{{ name }}]="{{ url }}"
|
|
{% endfor %}
|
|
)
|
|
|
|
log() { echo "restic-backup: $*"; }
|
|
|
|
# Last good backup per repository, for Prometheus (node_exporter textfile)
|
|
metric() {
|
|
local f={{ monitoring_textfile_dir }}/restic_$1.prom
|
|
printf '# HELP restic_backup_last_success_timestamp_seconds Last successful restic backup.\n# TYPE restic_backup_last_success_timestamp_seconds gauge\nrestic_backup_last_success_timestamp_seconds{repo="%s"} %s\n' "$1" "$(date +%s)" > "$f.tmp" && mv "$f.tmp" "$f"
|
|
}
|
|
|
|
case "${1:-}" in
|
|
init)
|
|
for name in "${!REPOS[@]}"; do
|
|
if RESTIC_REPOSITORY="${REPOS[$name]}" restic cat config >/dev/null 2>&1; then
|
|
log "$name: exists"
|
|
else
|
|
RESTIC_REPOSITORY="${REPOS[$name]}" restic init || exit 1
|
|
log "$name: created"
|
|
fi
|
|
done
|
|
;;
|
|
|
|
run)
|
|
failed=0
|
|
rm -rf "$STAGE"; install -d -m 700 "$STAGE"
|
|
export STAGE
|
|
{% for hook in restic_pre_hooks %}
|
|
log "hook {{ loop.index }}/{{ restic_pre_hooks | length }}"
|
|
( set -e; {{ hook }} ) || { log "hook failed"; failed=1; }
|
|
{% endfor %}
|
|
|
|
for name in "${!REPOS[@]}"; do
|
|
export RESTIC_REPOSITORY="${REPOS[$name]}"
|
|
log "$name: backup"
|
|
restic backup --host {{ inventory_hostname }} --one-file-system --exclude-caches \
|
|
{% for e in restic_excludes %}
|
|
--exclude '{{ e }}' \
|
|
{% endfor %}
|
|
{{ (restic_paths + [restic_stage_dir]) | join(' ') }} || { failed=1; continue; }
|
|
|
|
prune=""
|
|
[ "$(date +%a)" = "{{ restic_prune_weekday }}" ] && prune="--prune"
|
|
log "$name: forget $prune"
|
|
restic forget --host {{ inventory_hostname }} {{ restic_keep }} $prune || failed=1
|
|
metric "$name"
|
|
done
|
|
|
|
rm -rf "$STAGE"
|
|
exit $failed
|
|
;;
|
|
|
|
*)
|
|
[ -n "${1:-}" ] && [ -n "${REPOS[$1]:-}" ] || { sed -n '4,10p' "$0"; exit 2; }
|
|
export RESTIC_REPOSITORY="${REPOS[$1]}"; shift
|
|
exec restic "$@"
|
|
;;
|
|
esac
|