xmpp_cert_sync: push asxp.io and xmpp.asxp.io as separate files

Traefik never requests a certificate for a name it already holds one
for, so the XMPP certificate came out without asxp.io, the XMPP domain
itself. ejabberd picks the matching certificate file per domain, so push
two: the site's asxp.io certificate and xmpp.asxp.io + conference.
The receiver takes the name from the SSH command (asxp.io or a
subdomain), checks the certificate matches it, and retires the
hand-made server.pem once both files are in place.

A second probe on 5223 sends SNI asxp.io, as clients do, and the
outdated-certificate alert compares each served certificate with its
source in acme.json.

Assisted-by: Claude:opus-5-5

ansible/inventory/group_vars/all/xmpp_cert.yaml

@@ -3,6 +3,11 @@
 # services forwards its challenges there, role acme_forward) and pushed to
 # ejabberd on asxpio-services on every renewal (role xmpp_cert_sync).
 xmpp_cert_main: xmpp.asxp.io
-xmpp_cert_sans: [asxp.io, conference.asxp.io]
+xmpp_cert_sans: [conference.asxp.io]
+# Certificates pushed to ejabberd, by their main domain in acme.json; it
+# picks the matching file per domain. asxp.io is the site's own
+# certificate: Traefik never requests a name it already holds a
+# certificate for, so asxp.io can't be part of the one above.
+xmpp_cert_push: [asxp.io, xmpp.asxp.io]
 # renovate: datasource=docker depName=nginx
 acme_forward_nginx_version: "1.31.8-alpine-slim"

ansible/roles/monitoring/defaults/main.yaml

@@ -20,6 +20,7 @@ monitoring_probes:
   # so a STARTTLS dialogue times out: plain connect here, TLS on 5223.
   - { name: xmpp c2s, target: "xmpp.asxp.io:5222", module: tcp_connect }
   - { name: xmpp c2s tls, target: "xmpp.asxp.io:5223", module: tls_connect }
+  - { name: xmpp c2s tls asxp.io, target: "xmpp.asxp.io:5223", module: tls_connect_asxpio }
   - { name: xmpp s2s, target: "xmpp.asxp.io:5269", module: tcp_connect }
   - { name: turn, target: "xmpp.asxp.io:3478", module: tcp_connect }
   - { name: xray, target: "{{ hostvars['asxpio-services'].public_ip }}:443", module: tcp_connect }

ansible/roles/monitoring/templates/alert-rules.yaml.j2

@@ -174,7 +174,7 @@ groups:
           - refId: A
             relativeTimeRange: { from: 900, to: 0 }
             datasourceUid: prometheus
-            model: { refId: A, expr: "(max(xmpp_cert_source_not_after_timestamp_seconds) - min(probe_ssl_earliest_cert_expiry{instance=\"xmpp.asxp.io:5223\"})) / 86400", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
+            model: { refId: A, expr: "((max(xmpp_cert_source_not_after_timestamp_seconds{cert=\"asxp.io\"}) - min(probe_ssl_earliest_cert_expiry{probe=\"xmpp c2s tls asxp.io\"})) / 86400 > 1) or ((max(xmpp_cert_source_not_after_timestamp_seconds{cert=\"xmpp.asxp.io\"}) - min(probe_ssl_earliest_cert_expiry{probe=\"xmpp c2s tls\"})) / 86400 > 1)", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
           - refId: C
             datasourceUid: __expr__
             model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [1] } } ] }

ansible/roles/monitoring/templates/blackbox.yml.j2

@@ -19,6 +19,15 @@ modules:
     tcp:
       preferred_ip_protocol: ip4
       tls: true
+  # XMPP clients name the XMPP domain (asxp.io), not the SRV host, in SNI
+  tls_connect_asxpio:
+    prober: tcp
+    timeout: 10s
+    tcp:
+      preferred_ip_protocol: ip4
+      tls: true
+      tls_config:
+        server_name: asxp.io
   tcp_connect:
     prober: tcp
     timeout: 10s

ansible/roles/traefik/templates/dynamic.yml.j2

@@ -24,9 +24,8 @@ http:
   routers:
     # Exists only to obtain the XMPP certificate (ejabberd on services);
     # role xmpp_cert_sync pushes it there on every renewal. No traffic
-    # reaches it: these names point at services. The rule must not match
-    # asxp.io, or its higher priority would take the site over; the
-    # certificate still covers it through tls.domains.
+    # reaches it: these names point at services. asxp.io is pushed from
+    # the site's own certificate.
     xmpp-cert:
       rule: "Host(`{{ xmpp_cert_main }}`)"
       entryPoints: [websecure]

ansible/roles/xmpp_cert_sync/files/xmpp-cert-install

@@ -1,20 +1,33 @@
 #!/bin/sh
 # Managed by Ansible (role xmpp_cert_sync). Forced command of the
-# xmpp_cert_sync key from asxpio-ops: read certificate chain + key on
-# stdin, check it, install it for ejabberd and reload.
+# xmpp_cert_sync key from asxpio-ops: read a certificate chain + key on
+# stdin, check it, install it as certs/<name>.pem for ejabberd (which
+# picks the matching file per domain) and reload.
 set -eu
 umask 022
+name=${SSH_ORIGINAL_COMMAND:-}
+case "$name" in
+  asxp.io|*.asxp.io) ;;
+  *) echo "refusing certificate name '$name'" >&2; exit 1 ;;
+esac
+case "$name" in *[!a-z0-9.-]*) echo "bad name" >&2; exit 1 ;; esac
 dir=/opt/xmpp/state/certs
-tmp=$(mktemp "$dir/.server.pem.XXXXXX")
+tmp=$(mktemp "$dir/.incoming.XXXXXX")
 trap 'rm -f "$tmp"' EXIT
 head -c 65536 > "$tmp"
-# A certificate valid for at least another day, with its matching key
+# Valid for at least another day, for this name, with its matching key
 openssl x509 -in "$tmp" -noout -checkend 86400 >/dev/null
+openssl x509 -in "$tmp" -noout -checkhost "$name" | grep -q "does match"
 cert_pub=$(openssl x509 -in "$tmp" -noout -pubkey | sha256sum)
 key_pub=$(openssl pkey -in "$tmp" -pubout | sha256sum)
 [ "$cert_pub" = "$key_pub" ] || { echo "key does not match certificate" >&2; exit 1; }
 chmod 0644 "$tmp"
-mv "$tmp" "$dir/server.pem"
+mv "$tmp" "$dir/$name.pem"
 trap - EXIT
+# The hand-made certificate from before the push; kept outside the glob
+if [ -f "$dir/server.pem" ] && [ -f "$dir/asxp.io.pem" ] && [ -f "$dir/xmpp.asxp.io.pem" ]; then
+  mkdir -p /opt/xmpp/state/certs-retired
+  mv "$dir/server.pem" "/opt/xmpp/state/certs-retired/server.pem.$(date +%Y%m%d)"
+fi
 docker compose -f /opt/xmpp/docker-compose.yml exec -T ejabberd ejabberdctl reload_config
-logger -t xmpp-cert-install "installed $(openssl x509 -in "$dir/server.pem" -noout -enddate)"
+logger -t xmpp-cert-install "installed $name.pem, $(openssl x509 -in "$dir/$name.pem" -noout -enddate)"

ansible/roles/xmpp_cert_sync/templates/xmpp-cert-sync.j2

@@ -1,60 +1,68 @@
 #!/usr/bin/env python3
 # Managed by Ansible (role xmpp_cert_sync).
-# Push the XMPP certificate from Traefik's acme.json to ejabberd on
-# asxpio-services whenever it changes (xmpp-cert-sync.path), and daily as
+# Push the XMPP certificates from Traefik's acme.json to ejabberd on
+# asxpio-services whenever they change (xmpp-cert-sync.path), and daily as
 # a fallback. The SSH key may only run xmpp-cert-install there.
 import base64, hashlib, json, os, subprocess, sys, tempfile, time
 
 ACME = "/opt/traefik/letsencrypt/acme.json"
-MAIN = "{{ xmpp_cert_main }}"
+CERTS = {{ xmpp_cert_push | to_json }}
 TARGET = "root@{{ asxpio_services_vpc_ip }}"
 KEY = "/root/.ssh/xmpp_cert_sync"
 KNOWN = "/root/.ssh/xmpp_cert_sync_known_hosts"
-STATE = "/var/lib/xmpp-cert-sync/pushed.sha256"
+STATE = "/var/lib/xmpp-cert-sync"
 PROM = "{{ monitoring_textfile_dir }}/xmpp_cert.prom"
 
 
-def metrics(**m):
-    lines = []
-    for name, (help_, value) in m.items():
-        lines += [f"# HELP {name} {help_}", f"# TYPE {name} gauge", f"{name} {value}"]
-    tmp = PROM + ".tmp"
-    with open(tmp, "w") as f:
-        f.write("\n".join(lines) + "\n")
-    os.replace(tmp, PROM)
+def not_after(pem):
+    with tempfile.NamedTemporaryFile() as t:
+        t.write(pem); t.flush()
+        end = subprocess.run(["openssl", "x509", "-in", t.name, "-noout", "-enddate"],
+                             capture_output=True, text=True, check=True).stdout
+    end = end.split("=", 1)[1].strip()
+    return end, subprocess.run(["date", "-d", end, "+%s"], capture_output=True,
+                               text=True, check=True).stdout.strip()
 
 
-certs = json.load(open(ACME))["letsencrypt"]["Certificates"] or []
-entry = next((c for c in certs if c["domain"]["main"] == MAIN), None)
-if not entry:
-    print(f"no certificate for {MAIN} in {ACME} yet")
-    sys.exit(0)
-pem = base64.b64decode(entry["certificate"]) + base64.b64decode(entry["key"])
-digest = hashlib.sha256(pem).hexdigest()
-
-with tempfile.NamedTemporaryFile() as t:
-    t.write(pem); t.flush()
-    end = subprocess.run(["openssl", "x509", "-in", t.name, "-noout", "-enddate"],
-                         capture_output=True, text=True, check=True).stdout
-not_after = subprocess.run(["date", "-d", end.split("=", 1)[1].strip(), "+%s"],
-                           capture_output=True, text=True, check=True).stdout.strip()
-
-try:
-    pushed = open(STATE).read().strip()
-except OSError:
-    pushed = ""
-if pushed != digest or "--force" in sys.argv:
+acme = json.load(open(ACME))["letsencrypt"]["Certificates"] or []
+os.makedirs(STATE, exist_ok=True)
+expiry, missing = {}, []
+for name in CERTS:
+    entry = next((c for c in acme if c["domain"]["main"] == name), None)
+    if not entry:
+        print(f"no certificate for {name} in {ACME} yet")
+        missing.append(name)
+        continue
+    pem = base64.b64decode(entry["certificate"]) + base64.b64decode(entry["key"])
+    end, expiry[name] = not_after(pem)
+    digest = hashlib.sha256(pem).hexdigest()
+    state = os.path.join(STATE, name + ".sha256")
+    try:
+        pushed = open(state).read().strip()
+    except OSError:
+        pushed = ""
+    if pushed == digest and "--force" not in sys.argv:
+        print(f"{name}: ejabberd already has it (valid until {end})")
+        continue
     subprocess.run(["ssh", "-i", KEY, "-o", "BatchMode=yes", "-o", "IdentitiesOnly=yes",
                     "-o", f"UserKnownHostsFile={KNOWN}", "-o", "StrictHostKeyChecking=accept-new",
-                    TARGET], input=pem, check=True, timeout=120)
-    os.makedirs(os.path.dirname(STATE), exist_ok=True)
-    with open(STATE, "w") as f:
+                    TARGET, name], input=pem, check=True, timeout=120)
+    with open(state, "w") as f:
         f.write(digest + "\n")
-    print(f"pushed {MAIN} (valid until {end.split('=', 1)[1].strip()})")
-else:
-    print("ejabberd already has the current certificate")
+    print(f"{name}: pushed (valid until {end})")
 
-metrics(
-    xmpp_cert_source_not_after_timestamp_seconds=("Expiry of the XMPP certificate in Traefik's acme.json.", not_after),
-    xmpp_cert_sync_last_success_timestamp_seconds=("Last run that left ejabberd with Traefik's current certificate.", int(time.time())),
-)
+lines = ["# HELP xmpp_cert_source_not_after_timestamp_seconds Expiry of each XMPP certificate in Traefik's acme.json.",
+         "# TYPE xmpp_cert_source_not_after_timestamp_seconds gauge"]
+lines += [f'xmpp_cert_source_not_after_timestamp_seconds{{cert="{n}"}} {t}' for n, t in expiry.items()]
+if not missing:
+    lines += ["# HELP xmpp_cert_sync_last_success_timestamp_seconds Last run that left ejabberd with all of Traefik's current certificates.",
+              "# TYPE xmpp_cert_sync_last_success_timestamp_seconds gauge",
+              f"xmpp_cert_sync_last_success_timestamp_seconds {int(time.time())}"]
+else:
+    try:  # keep the last success, so the stale alert still counts from it
+        lines += [l.rstrip("\n") for l in open(PROM) if l.startswith(("# HELP xmpp_cert_sync", "# TYPE xmpp_cert_sync", "xmpp_cert_sync"))]
+    except OSError:
+        pass
+with open(PROM + ".tmp", "w") as f:
+    f.write("\n".join(lines) + "\n")
+os.replace(PROM + ".tmp", PROM)