diff --git a/ansible/roles/monitoring/templates/alert-rules.yaml.j2 b/ansible/roles/monitoring/templates/alert-rules.yaml.j2 index e62fd89..63298e3 100644 --- a/ansible/roles/monitoring/templates/alert-rules.yaml.j2 +++ b/ansible/roles/monitoring/templates/alert-rules.yaml.j2 @@ -151,6 +151,38 @@ groups: for: 0m labels: { severity: warning } annotations: { summary: "restic on {{ $labels.host }}: last good backup {{ $values.A.Value | printf \"%.0f\" }} hours ago" } + - uid: xmpp-cert-sync-stale + title: "XMPP certificate sync stale" + condition: C + data: + - refId: A + relativeTimeRange: { from: 900, to: 0 } + datasourceUid: prometheus + model: { refId: A, expr: "(time() - xmpp_cert_sync_last_success_timestamp_seconds) / 3600", instant: true, intervalMs: 30000, maxDataPoints: 43200 } + - refId: C + datasourceUid: __expr__ + model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [50] } } ] } + noDataState: OK + execErrState: Error + for: 0m + labels: { severity: warning } + annotations: { summary: "xmpp-cert-sync on asxpio-ops last succeeded {{ $values.A.Value | printf \"%.0f\" }} hours ago (journalctl -u xmpp-cert-sync)" } + - uid: xmpp-cert-outdated + title: "XMPP serves an older certificate than Traefik has" + condition: C + data: + - refId: A + relativeTimeRange: { from: 900, to: 0 } + datasourceUid: prometheus + model: { refId: A, expr: "(max(xmpp_cert_source_not_after_timestamp_seconds) - min(probe_ssl_earliest_cert_expiry{instance=\"xmpp.asxp.io:5223\"})) / 86400", instant: true, intervalMs: 30000, maxDataPoints: 43200 } + - refId: C + datasourceUid: __expr__ + model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [1] } } ] } + noDataState: OK + execErrState: Error + for: 6h + labels: { severity: warning } + annotations: { summary: "ejabberd's certificate expires {{ $values.A.Value | printf \"%.0f\" }} days before the one Traefik holds: the push or the reload failed" } - uid: replica-stale title: "Bucket replica stale" condition: C diff --git a/ansible/roles/traefik/templates/dynamic.yml.j2 b/ansible/roles/traefik/templates/dynamic.yml.j2 index 1c0ccf2..b741653 100644 --- a/ansible/roles/traefik/templates/dynamic.yml.j2 +++ b/ansible/roles/traefik/templates/dynamic.yml.j2 @@ -20,3 +20,20 @@ http: average: 100 burst: 50 period: 1s + + routers: + # Exists only to obtain the XMPP certificate (ejabberd on services); + # role xmpp_cert_sync pushes it there on every renewal. No traffic + # reaches it: these names point at services. The rule must not match + # asxp.io, or its higher priority would take the site over; the + # certificate still covers it through tls.domains. + xmpp-cert: + rule: "Host(`{{ xmpp_cert_main }}`)" + entryPoints: [websecure] + priority: 1 + service: noop@internal + tls: + certResolver: letsencrypt + domains: + - main: {{ xmpp_cert_main }} + sans: {{ xmpp_cert_sans | to_json }} diff --git a/ansible/roles/xmpp/tasks/main.yaml b/ansible/roles/xmpp/tasks/main.yaml index 77d9c6b..248c3d9 100644 --- a/ansible/roles/xmpp/tasks/main.yaml +++ b/ansible/roles/xmpp/tasks/main.yaml @@ -2,7 +2,8 @@ # Compose and settings are managed here. The secrets and keys stay on the # host as data, like a game world, and are covered by the restic backup: # state/secrets.env ejabberd secrets -# state/certs/server.pem TLS certificate (asxp.io, conference., xmpp.) +# state/certs/server.pem TLS certificate (asxp.io, conference., xmpp.), issued by +# Traefik on ops and pushed by role xmpp_cert_sync # state/apushd/ push settings and VAPID key # state/tackygram/{env,key} gateway credentials and master key # Data: volumes xmpp_database, xmpp_tackygram. diff --git a/ansible/roles/xmpp_cert_sync/files/xmpp-cert-install b/ansible/roles/xmpp_cert_sync/files/xmpp-cert-install new file mode 100644 index 0000000..43d63f3 --- /dev/null +++ b/ansible/roles/xmpp_cert_sync/files/xmpp-cert-install @@ -0,0 +1,20 @@ +#!/bin/sh +# Managed by Ansible (role xmpp_cert_sync). Forced command of the +# xmpp_cert_sync key from asxpio-ops: read certificate chain + key on +# stdin, check it, install it for ejabberd and reload. +set -eu +umask 022 +dir=/opt/xmpp/state/certs +tmp=$(mktemp "$dir/.server.pem.XXXXXX") +trap 'rm -f "$tmp"' EXIT +head -c 65536 > "$tmp" +# A certificate valid for at least another day, with its matching key +openssl x509 -in "$tmp" -noout -checkend 86400 >/dev/null +cert_pub=$(openssl x509 -in "$tmp" -noout -pubkey | sha256sum) +key_pub=$(openssl pkey -in "$tmp" -pubout | sha256sum) +[ "$cert_pub" = "$key_pub" ] || { echo "key does not match certificate" >&2; exit 1; } +chmod 0644 "$tmp" +mv "$tmp" "$dir/server.pem" +trap - EXIT +docker compose -f /opt/xmpp/docker-compose.yml exec -T ejabberd ejabberdctl reload_config +logger -t xmpp-cert-install "installed $(openssl x509 -in "$dir/server.pem" -noout -enddate)" diff --git a/ansible/roles/xmpp_cert_sync/files/xmpp-cert-sync.path b/ansible/roles/xmpp_cert_sync/files/xmpp-cert-sync.path new file mode 100644 index 0000000..1034283 --- /dev/null +++ b/ansible/roles/xmpp_cert_sync/files/xmpp-cert-sync.path @@ -0,0 +1,8 @@ +[Unit] +Description=Watch Traefik's acme.json for a renewed XMPP certificate + +[Path] +PathChanged=/opt/traefik/letsencrypt/acme.json + +[Install] +WantedBy=paths.target diff --git a/ansible/roles/xmpp_cert_sync/files/xmpp-cert-sync.service b/ansible/roles/xmpp_cert_sync/files/xmpp-cert-sync.service new file mode 100644 index 0000000..54a1746 --- /dev/null +++ b/ansible/roles/xmpp_cert_sync/files/xmpp-cert-sync.service @@ -0,0 +1,7 @@ +[Unit] +Description=Push the XMPP certificate from Traefik to ejabberd +After=docker.service + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/xmpp-cert-sync diff --git a/ansible/roles/xmpp_cert_sync/files/xmpp-cert-sync.timer b/ansible/roles/xmpp_cert_sync/files/xmpp-cert-sync.timer new file mode 100644 index 0000000..9b5ab78 --- /dev/null +++ b/ansible/roles/xmpp_cert_sync/files/xmpp-cert-sync.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Daily XMPP certificate check (fallback for the path unit) + +[Timer] +OnCalendar=*-*-* 05:40:00 +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/xmpp_cert_sync/handlers/main.yaml b/ansible/roles/xmpp_cert_sync/handlers/main.yaml new file mode 100644 index 0000000..c28484f --- /dev/null +++ b/ansible/roles/xmpp_cert_sync/handlers/main.yaml @@ -0,0 +1,4 @@ +--- +- name: Reload systemd + ansible.builtin.systemd: + daemon_reload: true diff --git a/ansible/roles/xmpp_cert_sync/tasks/main.yaml b/ansible/roles/xmpp_cert_sync/tasks/main.yaml new file mode 100644 index 0000000..2a1a832 --- /dev/null +++ b/ansible/roles/xmpp_cert_sync/tasks/main.yaml @@ -0,0 +1,57 @@ +--- +# Runs on asxpio-ops (where Traefik keeps the certificate); sets up the +# receiving end on asxpio-services through delegate_to. +- name: Generate the sync SSH key + community.crypto.openssh_keypair: + path: /root/.ssh/xmpp_cert_sync + type: ed25519 + comment: xmpp-cert-sync@{{ inventory_hostname }} + register: xmpp_cert_sync_key + +- name: Install the receiving script on asxpio-services + ansible.builtin.copy: + src: xmpp-cert-install + dest: /usr/local/bin/xmpp-cert-install + mode: "0755" + delegate_to: asxpio-services + +- name: Authorize the key for that script only, from ops only + ansible.posix.authorized_key: + user: root + key: "{{ xmpp_cert_sync_key.public_key }}" + key_options: 'restrict,from="{{ asxpio_ops_vpc_ip }}",command="/usr/local/bin/xmpp-cert-install"' + comment: xmpp-cert-sync + delegate_to: asxpio-services + # In a dry run on a fresh host the key does not exist yet + when: xmpp_cert_sync_key.public_key is defined + +- name: Install the sync script + ansible.builtin.template: + src: xmpp-cert-sync.j2 + dest: /usr/local/bin/xmpp-cert-sync + mode: "0755" + +- name: Install the sync units + ansible.builtin.copy: + src: "{{ item }}" + dest: "/etc/systemd/system/{{ item }}" + mode: "0644" + loop: + - xmpp-cert-sync.service + - xmpp-cert-sync.path + - xmpp-cert-sync.timer + notify: Reload systemd + +- name: Flush handlers + ansible.builtin.meta: flush_handlers + +- name: Enable the path watch and the daily fallback + ansible.builtin.systemd: + name: "{{ item }}" + enabled: true + state: started + loop: + - xmpp-cert-sync.path + - xmpp-cert-sync.timer + # A dry run installs no new unit files + ignore_errors: "{{ ansible_check_mode }}" diff --git a/ansible/roles/xmpp_cert_sync/templates/xmpp-cert-sync.j2 b/ansible/roles/xmpp_cert_sync/templates/xmpp-cert-sync.j2 new file mode 100644 index 0000000..ce8f279 --- /dev/null +++ b/ansible/roles/xmpp_cert_sync/templates/xmpp-cert-sync.j2 @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +# Managed by Ansible (role xmpp_cert_sync). +# Push the XMPP certificate from Traefik's acme.json to ejabberd on +# asxpio-services whenever it changes (xmpp-cert-sync.path), and daily as +# a fallback. The SSH key may only run xmpp-cert-install there. +import base64, hashlib, json, os, subprocess, sys, tempfile, time + +ACME = "/opt/traefik/letsencrypt/acme.json" +MAIN = "{{ xmpp_cert_main }}" +TARGET = "root@{{ asxpio_services_vpc_ip }}" +KEY = "/root/.ssh/xmpp_cert_sync" +KNOWN = "/root/.ssh/xmpp_cert_sync_known_hosts" +STATE = "/var/lib/xmpp-cert-sync/pushed.sha256" +PROM = "{{ monitoring_textfile_dir }}/xmpp_cert.prom" + + +def metrics(**m): + lines = [] + for name, (help_, value) in m.items(): + lines += [f"# HELP {name} {help_}", f"# TYPE {name} gauge", f"{name} {value}"] + tmp = PROM + ".tmp" + with open(tmp, "w") as f: + f.write("\n".join(lines) + "\n") + os.replace(tmp, PROM) + + +certs = json.load(open(ACME))["letsencrypt"]["Certificates"] or [] +entry = next((c for c in certs if c["domain"]["main"] == MAIN), None) +if not entry: + print(f"no certificate for {MAIN} in {ACME} yet") + sys.exit(0) +pem = base64.b64decode(entry["certificate"]) + base64.b64decode(entry["key"]) +digest = hashlib.sha256(pem).hexdigest() + +with tempfile.NamedTemporaryFile() as t: + t.write(pem); t.flush() + end = subprocess.run(["openssl", "x509", "-in", t.name, "-noout", "-enddate"], + capture_output=True, text=True, check=True).stdout +not_after = subprocess.run(["date", "-d", end.split("=", 1)[1].strip(), "+%s"], + capture_output=True, text=True, check=True).stdout.strip() + +try: + pushed = open(STATE).read().strip() +except OSError: + pushed = "" +if pushed != digest or "--force" in sys.argv: + subprocess.run(["ssh", "-i", KEY, "-o", "BatchMode=yes", "-o", "IdentitiesOnly=yes", + "-o", f"UserKnownHostsFile={KNOWN}", "-o", "StrictHostKeyChecking=accept-new", + TARGET], input=pem, check=True, timeout=120) + os.makedirs(os.path.dirname(STATE), exist_ok=True) + with open(STATE, "w") as f: + f.write(digest + "\n") + print(f"pushed {MAIN} (valid until {end.split('=', 1)[1].strip()})") +else: + print("ejabberd already has the current certificate") + +metrics( + xmpp_cert_source_not_after_timestamp_seconds=("Expiry of the XMPP certificate in Traefik's acme.json.", not_after), + xmpp_cert_sync_last_success_timestamp_seconds=("Last run that left ejabberd with Traefik's current certificate.", int(time.time())), +) diff --git a/ansible/site.yaml b/ansible/site.yaml index c385845..705159e 100644 --- a/ansible/site.yaml +++ b/ansible/site.yaml @@ -25,6 +25,7 @@ become: true roles: - { role: traefik, tags: [traefik] } + - { role: xmpp_cert_sync, tags: [xmpp_cert_sync, xmpp] } - { role: crawler_alert, tags: [crawler_alert] } - { role: monitoring, tags: [monitoring] }