xmpp_cert_sync: renew the XMPP certificate through Traefik

The ejabberd certificate (asxp.io, conference., xmpp.) was issued by
hand and expires 2027-01-28 with nothing renewing it. Traefik now
obtains it alongside the sites' certificates, through a router that
only exists to request it, and a systemd path unit on ops pushes it to
services whenever acme.json changes, with a daily run as fallback.

The push uses its own SSH key, accepted from ops' VPC address only and
forced to xmpp-cert-install, which checks the certificate is valid for
another day and matches its key before replacing server.pem and
reloading ejabberd.

Two alerts: the sync hasn't succeeded for two days, or ejabberd serves
an older certificate than Traefik holds for six hours. The existing
expiry alert already covers port 5223.

Assisted-by: Claude:opus-5-5

ansible/roles/xmpp_cert_sync/templates/xmpp-cert-sync.j2

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+# Managed by Ansible (role xmpp_cert_sync).
+# Push the XMPP certificate from Traefik's acme.json to ejabberd on
+# asxpio-services whenever it changes (xmpp-cert-sync.path), and daily as
+# a fallback. The SSH key may only run xmpp-cert-install there.
+import base64, hashlib, json, os, subprocess, sys, tempfile, time
+
+ACME = "/opt/traefik/letsencrypt/acme.json"
+MAIN = "{{ xmpp_cert_main }}"
+TARGET = "root@{{ asxpio_services_vpc_ip }}"
+KEY = "/root/.ssh/xmpp_cert_sync"
+KNOWN = "/root/.ssh/xmpp_cert_sync_known_hosts"
+STATE = "/var/lib/xmpp-cert-sync/pushed.sha256"
+PROM = "{{ monitoring_textfile_dir }}/xmpp_cert.prom"
+
+
+def metrics(**m):
+    lines = []
+    for name, (help_, value) in m.items():
+        lines += [f"# HELP {name} {help_}", f"# TYPE {name} gauge", f"{name} {value}"]
+    tmp = PROM + ".tmp"
+    with open(tmp, "w") as f:
+        f.write("\n".join(lines) + "\n")
+    os.replace(tmp, PROM)
+
+
+certs = json.load(open(ACME))["letsencrypt"]["Certificates"] or []
+entry = next((c for c in certs if c["domain"]["main"] == MAIN), None)
+if not entry:
+    print(f"no certificate for {MAIN} in {ACME} yet")
+    sys.exit(0)
+pem = base64.b64decode(entry["certificate"]) + base64.b64decode(entry["key"])
+digest = hashlib.sha256(pem).hexdigest()
+
+with tempfile.NamedTemporaryFile() as t:
+    t.write(pem); t.flush()
+    end = subprocess.run(["openssl", "x509", "-in", t.name, "-noout", "-enddate"],
+                         capture_output=True, text=True, check=True).stdout
+not_after = subprocess.run(["date", "-d", end.split("=", 1)[1].strip(), "+%s"],
+                           capture_output=True, text=True, check=True).stdout.strip()
+
+try:
+    pushed = open(STATE).read().strip()
+except OSError:
+    pushed = ""
+if pushed != digest or "--force" in sys.argv:
+    subprocess.run(["ssh", "-i", KEY, "-o", "BatchMode=yes", "-o", "IdentitiesOnly=yes",
+                    "-o", f"UserKnownHostsFile={KNOWN}", "-o", "StrictHostKeyChecking=accept-new",
+                    TARGET], input=pem, check=True, timeout=120)
+    os.makedirs(os.path.dirname(STATE), exist_ok=True)
+    with open(STATE, "w") as f:
+        f.write(digest + "\n")
+    print(f"pushed {MAIN} (valid until {end.split('=', 1)[1].strip()})")
+else:
+    print("ejabberd already has the current certificate")
+
+metrics(
+    xmpp_cert_source_not_after_timestamp_seconds=("Expiry of the XMPP certificate in Traefik's acme.json.", not_after),
+    xmpp_cert_sync_last_success_timestamp_seconds=("Last run that left ejabberd with Traefik's current certificate.", int(time.time())),
+)