Sergei Poljanski
cf142c6e3b
Some checks failed
deploy / ansible (push) Failing after 59s
Traefik never requests a certificate for a name it already holds one for, so the XMPP certificate came out without asxp.io, the XMPP domain itself. ejabberd picks the matching certificate file per domain, so push two: the site's asxp.io certificate and xmpp.asxp.io + conference. The receiver takes the name from the SSH command (asxp.io or a subdomain), checks the certificate matches it, and retires the hand-made server.pem once both files are in place. A second probe on 5223 sends SNI asxp.io, as clients do, and the outdated-certificate alert compares each served certificate with its source in acme.json. Assisted-by: Claude:opus-5-5
202 lines
9.5 KiB
Django/Jinja
202 lines
9.5 KiB
Django/Jinja
# Managed by Ansible (role monitoring). Generated; see the commit adding it.
|
|
{% raw %}
|
|
apiVersion: 1
|
|
groups:
|
|
- orgId: 1
|
|
name: asxpio
|
|
folder: Alerts
|
|
interval: 1m
|
|
rules:
|
|
- uid: host-down
|
|
title: "Host down"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 900, to: 0 }
|
|
datasourceUid: prometheus
|
|
model: { refId: A, expr: "1 - up{job=\"node\"}", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [0] } } ] }
|
|
noDataState: OK
|
|
execErrState: Error
|
|
for: 2m
|
|
labels: { severity: critical }
|
|
annotations: { summary: "{{ $labels.host }} stopped answering (node_exporter over the VPC)" }
|
|
- uid: target-down
|
|
title: "Scrape target down"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 900, to: 0 }
|
|
datasourceUid: prometheus
|
|
model: { refId: A, expr: "up{job!~\"node|probe_.*\"} == 0", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [-1] } } ] }
|
|
noDataState: OK
|
|
execErrState: Error
|
|
for: 5m
|
|
labels: { severity: warning }
|
|
annotations: { summary: "{{ $labels.job }} on {{ $labels.instance }} is not answering metrics scrapes" }
|
|
- uid: probe-down
|
|
title: "Service down (outside-in probe)"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 900, to: 0 }
|
|
datasourceUid: prometheus
|
|
model: { refId: A, expr: "1 - probe_success", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [0] } } ] }
|
|
noDataState: OK
|
|
execErrState: Error
|
|
for: 2m
|
|
labels: { severity: critical }
|
|
annotations: { summary: "{{ $labels.probe }} ({{ $labels.instance }}) is not reachable" }
|
|
- uid: cert-expiry
|
|
title: "Certificate expires soon"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 900, to: 0 }
|
|
datasourceUid: prometheus
|
|
model: { refId: A, expr: "(probe_ssl_earliest_cert_expiry - time()) / 86400", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: lt, params: [14] } } ] }
|
|
noDataState: OK
|
|
execErrState: Error
|
|
for: 1h
|
|
labels: { severity: warning }
|
|
annotations: { summary: "{{ $labels.probe }}: certificate expires in {{ $values.A.Value | printf \"%.0f\" }} days" }
|
|
- uid: disk-full
|
|
title: "Disk almost full"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 900, to: 0 }
|
|
datasourceUid: prometheus
|
|
model: { refId: A, expr: "100 * (1 - node_filesystem_avail_bytes{fstype!~\"tmpfs|overlay|squashfs|fuse.*\"} / node_filesystem_size_bytes{fstype!~\"tmpfs|overlay|squashfs|fuse.*\"})", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [85] } } ] }
|
|
noDataState: OK
|
|
execErrState: Error
|
|
for: 10m
|
|
labels: { severity: warning }
|
|
annotations: { summary: "{{ $labels.host }} {{ $labels.mountpoint }} is {{ $values.A.Value | printf \"%.0f\" }}% full" }
|
|
- uid: memory-low
|
|
title: "Memory almost exhausted"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 900, to: 0 }
|
|
datasourceUid: prometheus
|
|
model: { refId: A, expr: "100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [92] } } ] }
|
|
noDataState: OK
|
|
execErrState: Error
|
|
for: 10m
|
|
labels: { severity: warning }
|
|
annotations: { summary: "{{ $labels.host }} memory is {{ $values.A.Value | printf \"%.0f\" }}% used" }
|
|
- uid: container-restarting
|
|
title: "Container restart-looping"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 900, to: 0 }
|
|
datasourceUid: prometheus
|
|
model: { refId: A, expr: "changes(container_start_time_seconds{name!=\"\"}[15m])", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [2] } } ] }
|
|
noDataState: OK
|
|
execErrState: Error
|
|
for: 0m
|
|
labels: { severity: critical }
|
|
annotations: { summary: "{{ $labels.name }} on {{ $labels.host }} restarted {{ $values.A.Value | printf \"%.0f\" }} times in 15 minutes" }
|
|
- uid: postgres-down
|
|
title: "Postgres down"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 900, to: 0 }
|
|
datasourceUid: prometheus
|
|
model: { refId: A, expr: "pg_up", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: lt, params: [1] } } ] }
|
|
noDataState: OK
|
|
execErrState: Error
|
|
for: 1m
|
|
labels: { severity: critical }
|
|
annotations: { summary: "Postgres (storage) is not answering" }
|
|
- uid: backup-stale
|
|
title: "Backup stale"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 900, to: 0 }
|
|
datasourceUid: prometheus
|
|
model: { refId: A, expr: "(time() - restic_backup_last_success_timestamp_seconds) / 3600", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [26] } } ] }
|
|
noDataState: OK
|
|
execErrState: Error
|
|
for: 0m
|
|
labels: { severity: warning }
|
|
annotations: { summary: "restic on {{ $labels.host }}: last good backup {{ $values.A.Value | printf \"%.0f\" }} hours ago" }
|
|
- uid: xmpp-cert-sync-stale
|
|
title: "XMPP certificate sync stale"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 900, to: 0 }
|
|
datasourceUid: prometheus
|
|
model: { refId: A, expr: "(time() - xmpp_cert_sync_last_success_timestamp_seconds) / 3600", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [50] } } ] }
|
|
noDataState: OK
|
|
execErrState: Error
|
|
for: 0m
|
|
labels: { severity: warning }
|
|
annotations: { summary: "xmpp-cert-sync on asxpio-ops last succeeded {{ $values.A.Value | printf \"%.0f\" }} hours ago (journalctl -u xmpp-cert-sync)" }
|
|
- uid: xmpp-cert-outdated
|
|
title: "XMPP serves an older certificate than Traefik has"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 900, to: 0 }
|
|
datasourceUid: prometheus
|
|
model: { refId: A, expr: "((max(xmpp_cert_source_not_after_timestamp_seconds{cert=\"asxp.io\"}) - min(probe_ssl_earliest_cert_expiry{probe=\"xmpp c2s tls asxp.io\"})) / 86400 > 1) or ((max(xmpp_cert_source_not_after_timestamp_seconds{cert=\"xmpp.asxp.io\"}) - min(probe_ssl_earliest_cert_expiry{probe=\"xmpp c2s tls\"})) / 86400 > 1)", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [1] } } ] }
|
|
noDataState: OK
|
|
execErrState: Error
|
|
for: 6h
|
|
labels: { severity: warning }
|
|
annotations: { summary: "ejabberd's certificate expires {{ $values.A.Value | printf \"%.0f\" }} days before the one Traefik holds: the push or the reload failed" }
|
|
- uid: replica-stale
|
|
title: "Bucket replica stale"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 900, to: 0 }
|
|
datasourceUid: prometheus
|
|
model: { refId: A, expr: "(time() - s3_replicate_last_success_timestamp_seconds) / 3600", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [26] } } ] }
|
|
noDataState: OK
|
|
execErrState: Error
|
|
for: 0m
|
|
labels: { severity: warning }
|
|
annotations: { summary: "hel1 -> fsn1 bucket replica: last good run {{ $values.A.Value | printf \"%.0f\" }} hours ago" }
|
|
{% endraw %}
|