From cf142c6e3ba55235e7ad5999ca6e167e5d8ade18 Mon Sep 17 00:00:00 2001 From: Sergei Poljanski Date: Fri, 13 Nov 2026 08:26:08 +0400 Subject: [PATCH] xmpp_cert_sync: push asxp.io and xmpp.asxp.io as separate files Traefik never requests a certificate for a name it already holds one for, so the XMPP certificate came out without asxp.io, the XMPP domain itself. ejabberd picks the matching certificate file per domain, so push two: the site's asxp.io certificate and xmpp.asxp.io + conference. The receiver takes the name from the SSH command (asxp.io or a subdomain), checks the certificate matches it, and retires the hand-made server.pem once both files are in place. A second probe on 5223 sends SNI asxp.io, as clients do, and the outdated-certificate alert compares each served certificate with its source in acme.json. Assisted-by: Claude:opus-5-5 --- .../inventory/group_vars/all/xmpp_cert.yaml | 7 +- ansible/roles/monitoring/defaults/main.yaml | 1 + .../monitoring/templates/alert-rules.yaml.j2 | 2 +- .../monitoring/templates/blackbox.yml.j2 | 9 ++ .../roles/traefik/templates/dynamic.yml.j2 | 5 +- .../xmpp_cert_sync/files/xmpp-cert-install | 25 +++-- .../templates/xmpp-cert-sync.j2 | 92 ++++++++++--------- 7 files changed, 88 insertions(+), 53 deletions(-) diff --git a/ansible/inventory/group_vars/all/xmpp_cert.yaml b/ansible/inventory/group_vars/all/xmpp_cert.yaml index 026c7f6..57c643d 100644 --- a/ansible/inventory/group_vars/all/xmpp_cert.yaml +++ b/ansible/inventory/group_vars/all/xmpp_cert.yaml @@ -3,6 +3,11 @@ # services forwards its challenges there, role acme_forward) and pushed to # ejabberd on asxpio-services on every renewal (role xmpp_cert_sync). xmpp_cert_main: xmpp.asxp.io -xmpp_cert_sans: [asxp.io, conference.asxp.io] +xmpp_cert_sans: [conference.asxp.io] +# Certificates pushed to ejabberd, by their main domain in acme.json; it +# picks the matching file per domain. asxp.io is the site's own +# certificate: Traefik never requests a name it already holds a +# certificate for, so asxp.io can't be part of the one above. +xmpp_cert_push: [asxp.io, xmpp.asxp.io] # renovate: datasource=docker depName=nginx acme_forward_nginx_version: "1.31.8-alpine-slim" diff --git a/ansible/roles/monitoring/defaults/main.yaml b/ansible/roles/monitoring/defaults/main.yaml index 5cd27c7..2d79387 100644 --- a/ansible/roles/monitoring/defaults/main.yaml +++ b/ansible/roles/monitoring/defaults/main.yaml @@ -20,6 +20,7 @@ monitoring_probes: # so a STARTTLS dialogue times out: plain connect here, TLS on 5223. - { name: xmpp c2s, target: "xmpp.asxp.io:5222", module: tcp_connect } - { name: xmpp c2s tls, target: "xmpp.asxp.io:5223", module: tls_connect } + - { name: xmpp c2s tls asxp.io, target: "xmpp.asxp.io:5223", module: tls_connect_asxpio } - { name: xmpp s2s, target: "xmpp.asxp.io:5269", module: tcp_connect } - { name: turn, target: "xmpp.asxp.io:3478", module: tcp_connect } - { name: xray, target: "{{ hostvars['asxpio-services'].public_ip }}:443", module: tcp_connect } diff --git a/ansible/roles/monitoring/templates/alert-rules.yaml.j2 b/ansible/roles/monitoring/templates/alert-rules.yaml.j2 index 63298e3..c6fb8ca 100644 --- a/ansible/roles/monitoring/templates/alert-rules.yaml.j2 +++ b/ansible/roles/monitoring/templates/alert-rules.yaml.j2 @@ -174,7 +174,7 @@ groups: - refId: A relativeTimeRange: { from: 900, to: 0 } datasourceUid: prometheus - model: { refId: A, expr: "(max(xmpp_cert_source_not_after_timestamp_seconds) - min(probe_ssl_earliest_cert_expiry{instance=\"xmpp.asxp.io:5223\"})) / 86400", instant: true, intervalMs: 30000, maxDataPoints: 43200 } + model: { refId: A, expr: "((max(xmpp_cert_source_not_after_timestamp_seconds{cert=\"asxp.io\"}) - min(probe_ssl_earliest_cert_expiry{probe=\"xmpp c2s tls asxp.io\"})) / 86400 > 1) or ((max(xmpp_cert_source_not_after_timestamp_seconds{cert=\"xmpp.asxp.io\"}) - min(probe_ssl_earliest_cert_expiry{probe=\"xmpp c2s tls\"})) / 86400 > 1)", instant: true, intervalMs: 30000, maxDataPoints: 43200 } - refId: C datasourceUid: __expr__ model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [1] } } ] } diff --git a/ansible/roles/monitoring/templates/blackbox.yml.j2 b/ansible/roles/monitoring/templates/blackbox.yml.j2 index 10286ee..27ad3f6 100644 --- a/ansible/roles/monitoring/templates/blackbox.yml.j2 +++ b/ansible/roles/monitoring/templates/blackbox.yml.j2 @@ -19,6 +19,15 @@ modules: tcp: preferred_ip_protocol: ip4 tls: true + # XMPP clients name the XMPP domain (asxp.io), not the SRV host, in SNI + tls_connect_asxpio: + prober: tcp + timeout: 10s + tcp: + preferred_ip_protocol: ip4 + tls: true + tls_config: + server_name: asxp.io tcp_connect: prober: tcp timeout: 10s diff --git a/ansible/roles/traefik/templates/dynamic.yml.j2 b/ansible/roles/traefik/templates/dynamic.yml.j2 index b741653..6a9c394 100644 --- a/ansible/roles/traefik/templates/dynamic.yml.j2 +++ b/ansible/roles/traefik/templates/dynamic.yml.j2 @@ -24,9 +24,8 @@ http: routers: # Exists only to obtain the XMPP certificate (ejabberd on services); # role xmpp_cert_sync pushes it there on every renewal. No traffic - # reaches it: these names point at services. The rule must not match - # asxp.io, or its higher priority would take the site over; the - # certificate still covers it through tls.domains. + # reaches it: these names point at services. asxp.io is pushed from + # the site's own certificate. xmpp-cert: rule: "Host(`{{ xmpp_cert_main }}`)" entryPoints: [websecure] diff --git a/ansible/roles/xmpp_cert_sync/files/xmpp-cert-install b/ansible/roles/xmpp_cert_sync/files/xmpp-cert-install index 43d63f3..b5b68d0 100644 --- a/ansible/roles/xmpp_cert_sync/files/xmpp-cert-install +++ b/ansible/roles/xmpp_cert_sync/files/xmpp-cert-install @@ -1,20 +1,33 @@ #!/bin/sh # Managed by Ansible (role xmpp_cert_sync). Forced command of the -# xmpp_cert_sync key from asxpio-ops: read certificate chain + key on -# stdin, check it, install it for ejabberd and reload. +# xmpp_cert_sync key from asxpio-ops: read a certificate chain + key on +# stdin, check it, install it as certs/.pem for ejabberd (which +# picks the matching file per domain) and reload. set -eu umask 022 +name=${SSH_ORIGINAL_COMMAND:-} +case "$name" in + asxp.io|*.asxp.io) ;; + *) echo "refusing certificate name '$name'" >&2; exit 1 ;; +esac +case "$name" in *[!a-z0-9.-]*) echo "bad name" >&2; exit 1 ;; esac dir=/opt/xmpp/state/certs -tmp=$(mktemp "$dir/.server.pem.XXXXXX") +tmp=$(mktemp "$dir/.incoming.XXXXXX") trap 'rm -f "$tmp"' EXIT head -c 65536 > "$tmp" -# A certificate valid for at least another day, with its matching key +# Valid for at least another day, for this name, with its matching key openssl x509 -in "$tmp" -noout -checkend 86400 >/dev/null +openssl x509 -in "$tmp" -noout -checkhost "$name" | grep -q "does match" cert_pub=$(openssl x509 -in "$tmp" -noout -pubkey | sha256sum) key_pub=$(openssl pkey -in "$tmp" -pubout | sha256sum) [ "$cert_pub" = "$key_pub" ] || { echo "key does not match certificate" >&2; exit 1; } chmod 0644 "$tmp" -mv "$tmp" "$dir/server.pem" +mv "$tmp" "$dir/$name.pem" trap - EXIT +# The hand-made certificate from before the push; kept outside the glob +if [ -f "$dir/server.pem" ] && [ -f "$dir/asxp.io.pem" ] && [ -f "$dir/xmpp.asxp.io.pem" ]; then + mkdir -p /opt/xmpp/state/certs-retired + mv "$dir/server.pem" "/opt/xmpp/state/certs-retired/server.pem.$(date +%Y%m%d)" +fi docker compose -f /opt/xmpp/docker-compose.yml exec -T ejabberd ejabberdctl reload_config -logger -t xmpp-cert-install "installed $(openssl x509 -in "$dir/server.pem" -noout -enddate)" +logger -t xmpp-cert-install "installed $name.pem, $(openssl x509 -in "$dir/$name.pem" -noout -enddate)" diff --git a/ansible/roles/xmpp_cert_sync/templates/xmpp-cert-sync.j2 b/ansible/roles/xmpp_cert_sync/templates/xmpp-cert-sync.j2 index ce8f279..bf1e80d 100644 --- a/ansible/roles/xmpp_cert_sync/templates/xmpp-cert-sync.j2 +++ b/ansible/roles/xmpp_cert_sync/templates/xmpp-cert-sync.j2 @@ -1,60 +1,68 @@ #!/usr/bin/env python3 # Managed by Ansible (role xmpp_cert_sync). -# Push the XMPP certificate from Traefik's acme.json to ejabberd on -# asxpio-services whenever it changes (xmpp-cert-sync.path), and daily as +# Push the XMPP certificates from Traefik's acme.json to ejabberd on +# asxpio-services whenever they change (xmpp-cert-sync.path), and daily as # a fallback. The SSH key may only run xmpp-cert-install there. import base64, hashlib, json, os, subprocess, sys, tempfile, time ACME = "/opt/traefik/letsencrypt/acme.json" -MAIN = "{{ xmpp_cert_main }}" +CERTS = {{ xmpp_cert_push | to_json }} TARGET = "root@{{ asxpio_services_vpc_ip }}" KEY = "/root/.ssh/xmpp_cert_sync" KNOWN = "/root/.ssh/xmpp_cert_sync_known_hosts" -STATE = "/var/lib/xmpp-cert-sync/pushed.sha256" +STATE = "/var/lib/xmpp-cert-sync" PROM = "{{ monitoring_textfile_dir }}/xmpp_cert.prom" -def metrics(**m): - lines = [] - for name, (help_, value) in m.items(): - lines += [f"# HELP {name} {help_}", f"# TYPE {name} gauge", f"{name} {value}"] - tmp = PROM + ".tmp" - with open(tmp, "w") as f: - f.write("\n".join(lines) + "\n") - os.replace(tmp, PROM) +def not_after(pem): + with tempfile.NamedTemporaryFile() as t: + t.write(pem); t.flush() + end = subprocess.run(["openssl", "x509", "-in", t.name, "-noout", "-enddate"], + capture_output=True, text=True, check=True).stdout + end = end.split("=", 1)[1].strip() + return end, subprocess.run(["date", "-d", end, "+%s"], capture_output=True, + text=True, check=True).stdout.strip() -certs = json.load(open(ACME))["letsencrypt"]["Certificates"] or [] -entry = next((c for c in certs if c["domain"]["main"] == MAIN), None) -if not entry: - print(f"no certificate for {MAIN} in {ACME} yet") - sys.exit(0) -pem = base64.b64decode(entry["certificate"]) + base64.b64decode(entry["key"]) -digest = hashlib.sha256(pem).hexdigest() - -with tempfile.NamedTemporaryFile() as t: - t.write(pem); t.flush() - end = subprocess.run(["openssl", "x509", "-in", t.name, "-noout", "-enddate"], - capture_output=True, text=True, check=True).stdout -not_after = subprocess.run(["date", "-d", end.split("=", 1)[1].strip(), "+%s"], - capture_output=True, text=True, check=True).stdout.strip() - -try: - pushed = open(STATE).read().strip() -except OSError: - pushed = "" -if pushed != digest or "--force" in sys.argv: +acme = json.load(open(ACME))["letsencrypt"]["Certificates"] or [] +os.makedirs(STATE, exist_ok=True) +expiry, missing = {}, [] +for name in CERTS: + entry = next((c for c in acme if c["domain"]["main"] == name), None) + if not entry: + print(f"no certificate for {name} in {ACME} yet") + missing.append(name) + continue + pem = base64.b64decode(entry["certificate"]) + base64.b64decode(entry["key"]) + end, expiry[name] = not_after(pem) + digest = hashlib.sha256(pem).hexdigest() + state = os.path.join(STATE, name + ".sha256") + try: + pushed = open(state).read().strip() + except OSError: + pushed = "" + if pushed == digest and "--force" not in sys.argv: + print(f"{name}: ejabberd already has it (valid until {end})") + continue subprocess.run(["ssh", "-i", KEY, "-o", "BatchMode=yes", "-o", "IdentitiesOnly=yes", "-o", f"UserKnownHostsFile={KNOWN}", "-o", "StrictHostKeyChecking=accept-new", - TARGET], input=pem, check=True, timeout=120) - os.makedirs(os.path.dirname(STATE), exist_ok=True) - with open(STATE, "w") as f: + TARGET, name], input=pem, check=True, timeout=120) + with open(state, "w") as f: f.write(digest + "\n") - print(f"pushed {MAIN} (valid until {end.split('=', 1)[1].strip()})") -else: - print("ejabberd already has the current certificate") + print(f"{name}: pushed (valid until {end})") -metrics( - xmpp_cert_source_not_after_timestamp_seconds=("Expiry of the XMPP certificate in Traefik's acme.json.", not_after), - xmpp_cert_sync_last_success_timestamp_seconds=("Last run that left ejabberd with Traefik's current certificate.", int(time.time())), -) +lines = ["# HELP xmpp_cert_source_not_after_timestamp_seconds Expiry of each XMPP certificate in Traefik's acme.json.", + "# TYPE xmpp_cert_source_not_after_timestamp_seconds gauge"] +lines += [f'xmpp_cert_source_not_after_timestamp_seconds{{cert="{n}"}} {t}' for n, t in expiry.items()] +if not missing: + lines += ["# HELP xmpp_cert_sync_last_success_timestamp_seconds Last run that left ejabberd with all of Traefik's current certificates.", + "# TYPE xmpp_cert_sync_last_success_timestamp_seconds gauge", + f"xmpp_cert_sync_last_success_timestamp_seconds {int(time.time())}"] +else: + try: # keep the last success, so the stale alert still counts from it + lines += [l.rstrip("\n") for l in open(PROM) if l.startswith(("# HELP xmpp_cert_sync", "# TYPE xmpp_cert_sync", "xmpp_cert_sync"))] + except OSError: + pass +with open(PROM + ".tmp", "w") as f: + f.write("\n".join(lines) + "\n") +os.replace(PROM + ".tmp", PROM)