xmpp_cert_sync: renew the XMPP certificate through Traefik

The ejabberd certificate (asxp.io, conference., xmpp.) was issued by
hand and expires 2027-01-28 with nothing renewing it. Traefik now
obtains it alongside the sites' certificates, through a router that
only exists to request it, and a systemd path unit on ops pushes it to
services whenever acme.json changes, with a daily run as fallback.

The push uses its own SSH key, accepted from ops' VPC address only and
forced to xmpp-cert-install, which checks the certificate is valid for
another day and matches its key before replacing server.pem and
reloading ejabberd.

Two alerts: the sync hasn't succeeded for two days, or ejabberd serves
an older certificate than Traefik holds for six hours. The existing
expiry alert already covers port 5223.

Assisted-by: Claude:opus-5-5

ansible/roles/monitoring/templates/alert-rules.yaml.j2

@@ -151,6 +151,38 @@ groups:
         for: 0m
         labels: { severity: warning }
         annotations: { summary: "restic on {{ $labels.host }}: last good backup {{ $values.A.Value | printf \"%.0f\" }} hours ago" }
+      - uid: xmpp-cert-sync-stale
+        title: "XMPP certificate sync stale"
+        condition: C
+        data:
+          - refId: A
+            relativeTimeRange: { from: 900, to: 0 }
+            datasourceUid: prometheus
+            model: { refId: A, expr: "(time() - xmpp_cert_sync_last_success_timestamp_seconds) / 3600", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
+          - refId: C
+            datasourceUid: __expr__
+            model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [50] } } ] }
+        noDataState: OK
+        execErrState: Error
+        for: 0m
+        labels: { severity: warning }
+        annotations: { summary: "xmpp-cert-sync on asxpio-ops last succeeded {{ $values.A.Value | printf \"%.0f\" }} hours ago (journalctl -u xmpp-cert-sync)" }
+      - uid: xmpp-cert-outdated
+        title: "XMPP serves an older certificate than Traefik has"
+        condition: C
+        data:
+          - refId: A
+            relativeTimeRange: { from: 900, to: 0 }
+            datasourceUid: prometheus
+            model: { refId: A, expr: "(max(xmpp_cert_source_not_after_timestamp_seconds) - min(probe_ssl_earliest_cert_expiry{instance=\"xmpp.asxp.io:5223\"})) / 86400", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
+          - refId: C
+            datasourceUid: __expr__
+            model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [1] } } ] }
+        noDataState: OK
+        execErrState: Error
+        for: 6h
+        labels: { severity: warning }
+        annotations: { summary: "ejabberd's certificate expires {{ $values.A.Value | printf \"%.0f\" }} days before the one Traefik holds: the push or the reload failed" }
       - uid: replica-stale
         title: "Bucket replica stale"
         condition: C

ansible/roles/traefik/templates/dynamic.yml.j2

@@ -20,3 +20,20 @@ http:
         average: 100
         burst: 50
         period: 1s
+
+  routers:
+    # Exists only to obtain the XMPP certificate (ejabberd on services);
+    # role xmpp_cert_sync pushes it there on every renewal. No traffic
+    # reaches it: these names point at services. The rule must not match
+    # asxp.io, or its higher priority would take the site over; the
+    # certificate still covers it through tls.domains.
+    xmpp-cert:
+      rule: "Host(`{{ xmpp_cert_main }}`)"
+      entryPoints: [websecure]
+      priority: 1
+      service: noop@internal
+      tls:
+        certResolver: letsencrypt
+        domains:
+          - main: {{ xmpp_cert_main }}
+            sans: {{ xmpp_cert_sans | to_json }}

ansible/roles/xmpp/tasks/main.yaml

@@ -2,7 +2,8 @@
 # Compose and settings are managed here. The secrets and keys stay on the
 # host as data, like a game world, and are covered by the restic backup:
 #   state/secrets.env          ejabberd secrets
-#   state/certs/server.pem     TLS certificate (asxp.io, conference., xmpp.)
+#   state/certs/server.pem     TLS certificate (asxp.io, conference., xmpp.), issued by
+#                              Traefik on ops and pushed by role xmpp_cert_sync
 #   state/apushd/              push settings and VAPID key
 #   state/tackygram/{env,key}  gateway credentials and master key
 # Data: volumes xmpp_database, xmpp_tackygram.

ansible/roles/xmpp_cert_sync/files/xmpp-cert-install

@@ -0,0 +1,20 @@
+#!/bin/sh
+# Managed by Ansible (role xmpp_cert_sync). Forced command of the
+# xmpp_cert_sync key from asxpio-ops: read certificate chain + key on
+# stdin, check it, install it for ejabberd and reload.
+set -eu
+umask 022
+dir=/opt/xmpp/state/certs
+tmp=$(mktemp "$dir/.server.pem.XXXXXX")
+trap 'rm -f "$tmp"' EXIT
+head -c 65536 > "$tmp"
+# A certificate valid for at least another day, with its matching key
+openssl x509 -in "$tmp" -noout -checkend 86400 >/dev/null
+cert_pub=$(openssl x509 -in "$tmp" -noout -pubkey | sha256sum)
+key_pub=$(openssl pkey -in "$tmp" -pubout | sha256sum)
+[ "$cert_pub" = "$key_pub" ] || { echo "key does not match certificate" >&2; exit 1; }
+chmod 0644 "$tmp"
+mv "$tmp" "$dir/server.pem"
+trap - EXIT
+docker compose -f /opt/xmpp/docker-compose.yml exec -T ejabberd ejabberdctl reload_config
+logger -t xmpp-cert-install "installed $(openssl x509 -in "$dir/server.pem" -noout -enddate)"

ansible/roles/xmpp_cert_sync/files/xmpp-cert-sync.path

@@ -0,0 +1,8 @@
+[Unit]
+Description=Watch Traefik's acme.json for a renewed XMPP certificate
+
+[Path]
+PathChanged=/opt/traefik/letsencrypt/acme.json
+
+[Install]
+WantedBy=paths.target

ansible/roles/xmpp_cert_sync/files/xmpp-cert-sync.service

@@ -0,0 +1,7 @@
+[Unit]
+Description=Push the XMPP certificate from Traefik to ejabberd
+After=docker.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/xmpp-cert-sync

ansible/roles/xmpp_cert_sync/files/xmpp-cert-sync.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Daily XMPP certificate check (fallback for the path unit)
+
+[Timer]
+OnCalendar=*-*-* 05:40:00
+Persistent=true
+
+[Install]
+WantedBy=timers.target

ansible/roles/xmpp_cert_sync/handlers/main.yaml

@@ -0,0 +1,4 @@
+---
+- name: Reload systemd
+  ansible.builtin.systemd:
+    daemon_reload: true

ansible/roles/xmpp_cert_sync/tasks/main.yaml

@@ -0,0 +1,57 @@
+---
+# Runs on asxpio-ops (where Traefik keeps the certificate); sets up the
+# receiving end on asxpio-services through delegate_to.
+- name: Generate the sync SSH key
+  community.crypto.openssh_keypair:
+    path: /root/.ssh/xmpp_cert_sync
+    type: ed25519
+    comment: xmpp-cert-sync@{{ inventory_hostname }}
+  register: xmpp_cert_sync_key
+
+- name: Install the receiving script on asxpio-services
+  ansible.builtin.copy:
+    src: xmpp-cert-install
+    dest: /usr/local/bin/xmpp-cert-install
+    mode: "0755"
+  delegate_to: asxpio-services
+
+- name: Authorize the key for that script only, from ops only
+  ansible.posix.authorized_key:
+    user: root
+    key: "{{ xmpp_cert_sync_key.public_key }}"
+    key_options: 'restrict,from="{{ asxpio_ops_vpc_ip }}",command="/usr/local/bin/xmpp-cert-install"'
+    comment: xmpp-cert-sync
+  delegate_to: asxpio-services
+  # In a dry run on a fresh host the key does not exist yet
+  when: xmpp_cert_sync_key.public_key is defined
+
+- name: Install the sync script
+  ansible.builtin.template:
+    src: xmpp-cert-sync.j2
+    dest: /usr/local/bin/xmpp-cert-sync
+    mode: "0755"
+
+- name: Install the sync units
+  ansible.builtin.copy:
+    src: "{{ item }}"
+    dest: "/etc/systemd/system/{{ item }}"
+    mode: "0644"
+  loop:
+    - xmpp-cert-sync.service
+    - xmpp-cert-sync.path
+    - xmpp-cert-sync.timer
+  notify: Reload systemd
+
+- name: Flush handlers
+  ansible.builtin.meta: flush_handlers
+
+- name: Enable the path watch and the daily fallback
+  ansible.builtin.systemd:
+    name: "{{ item }}"
+    enabled: true
+    state: started
+  loop:
+    - xmpp-cert-sync.path
+    - xmpp-cert-sync.timer
+  # A dry run installs no new unit files
+  ignore_errors: "{{ ansible_check_mode }}"

ansible/roles/xmpp_cert_sync/templates/xmpp-cert-sync.j2

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+# Managed by Ansible (role xmpp_cert_sync).
+# Push the XMPP certificate from Traefik's acme.json to ejabberd on
+# asxpio-services whenever it changes (xmpp-cert-sync.path), and daily as
+# a fallback. The SSH key may only run xmpp-cert-install there.
+import base64, hashlib, json, os, subprocess, sys, tempfile, time
+
+ACME = "/opt/traefik/letsencrypt/acme.json"
+MAIN = "{{ xmpp_cert_main }}"
+TARGET = "root@{{ asxpio_services_vpc_ip }}"
+KEY = "/root/.ssh/xmpp_cert_sync"
+KNOWN = "/root/.ssh/xmpp_cert_sync_known_hosts"
+STATE = "/var/lib/xmpp-cert-sync/pushed.sha256"
+PROM = "{{ monitoring_textfile_dir }}/xmpp_cert.prom"
+
+
+def metrics(**m):
+    lines = []
+    for name, (help_, value) in m.items():
+        lines += [f"# HELP {name} {help_}", f"# TYPE {name} gauge", f"{name} {value}"]
+    tmp = PROM + ".tmp"
+    with open(tmp, "w") as f:
+        f.write("\n".join(lines) + "\n")
+    os.replace(tmp, PROM)
+
+
+certs = json.load(open(ACME))["letsencrypt"]["Certificates"] or []
+entry = next((c for c in certs if c["domain"]["main"] == MAIN), None)
+if not entry:
+    print(f"no certificate for {MAIN} in {ACME} yet")
+    sys.exit(0)
+pem = base64.b64decode(entry["certificate"]) + base64.b64decode(entry["key"])
+digest = hashlib.sha256(pem).hexdigest()
+
+with tempfile.NamedTemporaryFile() as t:
+    t.write(pem); t.flush()
+    end = subprocess.run(["openssl", "x509", "-in", t.name, "-noout", "-enddate"],
+                         capture_output=True, text=True, check=True).stdout
+not_after = subprocess.run(["date", "-d", end.split("=", 1)[1].strip(), "+%s"],
+                           capture_output=True, text=True, check=True).stdout.strip()
+
+try:
+    pushed = open(STATE).read().strip()
+except OSError:
+    pushed = ""
+if pushed != digest or "--force" in sys.argv:
+    subprocess.run(["ssh", "-i", KEY, "-o", "BatchMode=yes", "-o", "IdentitiesOnly=yes",
+                    "-o", f"UserKnownHostsFile={KNOWN}", "-o", "StrictHostKeyChecking=accept-new",
+                    TARGET], input=pem, check=True, timeout=120)
+    os.makedirs(os.path.dirname(STATE), exist_ok=True)
+    with open(STATE, "w") as f:
+        f.write(digest + "\n")
+    print(f"pushed {MAIN} (valid until {end.split('=', 1)[1].strip()})")
+else:
+    print("ejabberd already has the current certificate")
+
+metrics(
+    xmpp_cert_source_not_after_timestamp_seconds=("Expiry of the XMPP certificate in Traefik's acme.json.", not_after),
+    xmpp_cert_sync_last_success_timestamp_seconds=("Last run that left ejabberd with Traefik's current certificate.", int(time.time())),
+)