Sergei Poljanski
5e009c4a47
All checks were successful
deploy / ansible (push) Successful in 7m22s
The ejabberd certificate (asxp.io, conference., xmpp.) was issued by hand and expires 2027-01-28 with nothing renewing it. Traefik now obtains it alongside the sites' certificates, through a router that only exists to request it, and a systemd path unit on ops pushes it to services whenever acme.json changes, with a daily run as fallback. The push uses its own SSH key, accepted from ops' VPC address only and forced to xmpp-cert-install, which checks the certificate is valid for another day and matches its key before replacing server.pem and reloading ejabberd. Two alerts: the sync hasn't succeeded for two days, or ejabberd serves an older certificate than Traefik holds for six hours. The existing expiry alert already covers port 5223. Assisted-by: Claude:opus-5-5
202 lines
9.4 KiB
Django/Jinja
202 lines
9.4 KiB
Django/Jinja
# Managed by Ansible (role monitoring). Generated; see the commit adding it.
|
|
{% raw %}
|
|
apiVersion: 1
|
|
groups:
|
|
- orgId: 1
|
|
name: asxpio
|
|
folder: Alerts
|
|
interval: 1m
|
|
rules:
|
|
- uid: host-down
|
|
title: "Host down"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 900, to: 0 }
|
|
datasourceUid: prometheus
|
|
model: { refId: A, expr: "1 - up{job=\"node\"}", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [0] } } ] }
|
|
noDataState: OK
|
|
execErrState: Error
|
|
for: 2m
|
|
labels: { severity: critical }
|
|
annotations: { summary: "{{ $labels.host }} stopped answering (node_exporter over the VPC)" }
|
|
- uid: target-down
|
|
title: "Scrape target down"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 900, to: 0 }
|
|
datasourceUid: prometheus
|
|
model: { refId: A, expr: "up{job!~\"node|probe_.*\"} == 0", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [-1] } } ] }
|
|
noDataState: OK
|
|
execErrState: Error
|
|
for: 5m
|
|
labels: { severity: warning }
|
|
annotations: { summary: "{{ $labels.job }} on {{ $labels.instance }} is not answering metrics scrapes" }
|
|
- uid: probe-down
|
|
title: "Service down (outside-in probe)"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 900, to: 0 }
|
|
datasourceUid: prometheus
|
|
model: { refId: A, expr: "1 - probe_success", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [0] } } ] }
|
|
noDataState: OK
|
|
execErrState: Error
|
|
for: 2m
|
|
labels: { severity: critical }
|
|
annotations: { summary: "{{ $labels.probe }} ({{ $labels.instance }}) is not reachable" }
|
|
- uid: cert-expiry
|
|
title: "Certificate expires soon"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 900, to: 0 }
|
|
datasourceUid: prometheus
|
|
model: { refId: A, expr: "(probe_ssl_earliest_cert_expiry - time()) / 86400", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: lt, params: [14] } } ] }
|
|
noDataState: OK
|
|
execErrState: Error
|
|
for: 1h
|
|
labels: { severity: warning }
|
|
annotations: { summary: "{{ $labels.probe }}: certificate expires in {{ $values.A.Value | printf \"%.0f\" }} days" }
|
|
- uid: disk-full
|
|
title: "Disk almost full"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 900, to: 0 }
|
|
datasourceUid: prometheus
|
|
model: { refId: A, expr: "100 * (1 - node_filesystem_avail_bytes{fstype!~\"tmpfs|overlay|squashfs|fuse.*\"} / node_filesystem_size_bytes{fstype!~\"tmpfs|overlay|squashfs|fuse.*\"})", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [85] } } ] }
|
|
noDataState: OK
|
|
execErrState: Error
|
|
for: 10m
|
|
labels: { severity: warning }
|
|
annotations: { summary: "{{ $labels.host }} {{ $labels.mountpoint }} is {{ $values.A.Value | printf \"%.0f\" }}% full" }
|
|
- uid: memory-low
|
|
title: "Memory almost exhausted"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 900, to: 0 }
|
|
datasourceUid: prometheus
|
|
model: { refId: A, expr: "100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [92] } } ] }
|
|
noDataState: OK
|
|
execErrState: Error
|
|
for: 10m
|
|
labels: { severity: warning }
|
|
annotations: { summary: "{{ $labels.host }} memory is {{ $values.A.Value | printf \"%.0f\" }}% used" }
|
|
- uid: container-restarting
|
|
title: "Container restart-looping"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 900, to: 0 }
|
|
datasourceUid: prometheus
|
|
model: { refId: A, expr: "changes(container_start_time_seconds{name!=\"\"}[15m])", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [2] } } ] }
|
|
noDataState: OK
|
|
execErrState: Error
|
|
for: 0m
|
|
labels: { severity: critical }
|
|
annotations: { summary: "{{ $labels.name }} on {{ $labels.host }} restarted {{ $values.A.Value | printf \"%.0f\" }} times in 15 minutes" }
|
|
- uid: postgres-down
|
|
title: "Postgres down"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 900, to: 0 }
|
|
datasourceUid: prometheus
|
|
model: { refId: A, expr: "pg_up", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: lt, params: [1] } } ] }
|
|
noDataState: OK
|
|
execErrState: Error
|
|
for: 1m
|
|
labels: { severity: critical }
|
|
annotations: { summary: "Postgres (storage) is not answering" }
|
|
- uid: backup-stale
|
|
title: "Backup stale"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 900, to: 0 }
|
|
datasourceUid: prometheus
|
|
model: { refId: A, expr: "(time() - restic_backup_last_success_timestamp_seconds) / 3600", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [26] } } ] }
|
|
noDataState: OK
|
|
execErrState: Error
|
|
for: 0m
|
|
labels: { severity: warning }
|
|
annotations: { summary: "restic on {{ $labels.host }}: last good backup {{ $values.A.Value | printf \"%.0f\" }} hours ago" }
|
|
- uid: xmpp-cert-sync-stale
|
|
title: "XMPP certificate sync stale"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 900, to: 0 }
|
|
datasourceUid: prometheus
|
|
model: { refId: A, expr: "(time() - xmpp_cert_sync_last_success_timestamp_seconds) / 3600", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [50] } } ] }
|
|
noDataState: OK
|
|
execErrState: Error
|
|
for: 0m
|
|
labels: { severity: warning }
|
|
annotations: { summary: "xmpp-cert-sync on asxpio-ops last succeeded {{ $values.A.Value | printf \"%.0f\" }} hours ago (journalctl -u xmpp-cert-sync)" }
|
|
- uid: xmpp-cert-outdated
|
|
title: "XMPP serves an older certificate than Traefik has"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 900, to: 0 }
|
|
datasourceUid: prometheus
|
|
model: { refId: A, expr: "(max(xmpp_cert_source_not_after_timestamp_seconds) - min(probe_ssl_earliest_cert_expiry{instance=\"xmpp.asxp.io:5223\"})) / 86400", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [1] } } ] }
|
|
noDataState: OK
|
|
execErrState: Error
|
|
for: 6h
|
|
labels: { severity: warning }
|
|
annotations: { summary: "ejabberd's certificate expires {{ $values.A.Value | printf \"%.0f\" }} days before the one Traefik holds: the push or the reload failed" }
|
|
- uid: replica-stale
|
|
title: "Bucket replica stale"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 900, to: 0 }
|
|
datasourceUid: prometheus
|
|
model: { refId: A, expr: "(time() - s3_replicate_last_success_timestamp_seconds) / 3600", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [26] } } ] }
|
|
noDataState: OK
|
|
execErrState: Error
|
|
for: 0m
|
|
labels: { severity: warning }
|
|
annotations: { summary: "hel1 -> fsn1 bucket replica: last good run {{ $values.A.Value | printf \"%.0f\" }} hours ago" }
|
|
{% endraw %}
|