Sergei Poljanski
e20f5a7d07
All checks were successful
deploy / ansible (push) Successful in 7m23s
Nothing watched the servers: tonight's asxpio crash loop was found by hand. On asxpio-ops, Prometheus (30 days), Loki (14 days) and Grafana at gf.asxp.io (WireGuard only), with blackbox_exporter probing every service from outside and postgres_exporter. On both hosts node_exporter and cAdvisor on the VPC address, and Alloy shipping container logs, the journal and Traefik's access log to Loki over the VPC. Traefik and Forgejo expose their metrics; restic and the bucket replica record their last success for node_exporter. Dashboards: an overview as home, plus hosts, containers, Traefik, probes, Postgres and logs. Alerts by email to me@asxp.io: host or service down, certificate under 14 days, disk or memory nearly full, a container restart-looping, Postgres down, backup or replica stale. Assisted-by: Claude:opus-5-5
170 lines
7.6 KiB
Django/Jinja
170 lines
7.6 KiB
Django/Jinja
# Managed by Ansible (role monitoring). Generated; see the commit adding it.
|
|
{% raw %}
|
|
apiVersion: 1
|
|
groups:
|
|
- orgId: 1
|
|
name: asxpio
|
|
folder: Alerts
|
|
interval: 1m
|
|
rules:
|
|
- uid: host-down
|
|
title: "Host down"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 900, to: 0 }
|
|
datasourceUid: prometheus
|
|
model: { refId: A, expr: "1 - up{job=\"node\"}", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [0] } } ] }
|
|
noDataState: OK
|
|
execErrState: Error
|
|
for: 2m
|
|
labels: { severity: critical }
|
|
annotations: { summary: "{{ $labels.host }} stopped answering (node_exporter over the VPC)" }
|
|
- uid: target-down
|
|
title: "Scrape target down"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 900, to: 0 }
|
|
datasourceUid: prometheus
|
|
model: { refId: A, expr: "up{job!~\"node|probe_.*\"} == 0", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [-1] } } ] }
|
|
noDataState: OK
|
|
execErrState: Error
|
|
for: 5m
|
|
labels: { severity: warning }
|
|
annotations: { summary: "{{ $labels.job }} on {{ $labels.instance }} is not answering metrics scrapes" }
|
|
- uid: probe-down
|
|
title: "Service down (outside-in probe)"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 900, to: 0 }
|
|
datasourceUid: prometheus
|
|
model: { refId: A, expr: "1 - probe_success", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [0] } } ] }
|
|
noDataState: OK
|
|
execErrState: Error
|
|
for: 2m
|
|
labels: { severity: critical }
|
|
annotations: { summary: "{{ $labels.probe }} ({{ $labels.instance }}) is not reachable" }
|
|
- uid: cert-expiry
|
|
title: "Certificate expires soon"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 900, to: 0 }
|
|
datasourceUid: prometheus
|
|
model: { refId: A, expr: "(probe_ssl_earliest_cert_expiry - time()) / 86400", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: lt, params: [14] } } ] }
|
|
noDataState: OK
|
|
execErrState: Error
|
|
for: 1h
|
|
labels: { severity: warning }
|
|
annotations: { summary: "{{ $labels.probe }}: certificate expires in {{ $values.A.Value | printf \"%.0f\" }} days" }
|
|
- uid: disk-full
|
|
title: "Disk almost full"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 900, to: 0 }
|
|
datasourceUid: prometheus
|
|
model: { refId: A, expr: "100 * (1 - node_filesystem_avail_bytes{fstype!~\"tmpfs|overlay|squashfs|fuse.*\"} / node_filesystem_size_bytes{fstype!~\"tmpfs|overlay|squashfs|fuse.*\"})", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [85] } } ] }
|
|
noDataState: OK
|
|
execErrState: Error
|
|
for: 10m
|
|
labels: { severity: warning }
|
|
annotations: { summary: "{{ $labels.host }} {{ $labels.mountpoint }} is {{ $values.A.Value | printf \"%.0f\" }}% full" }
|
|
- uid: memory-low
|
|
title: "Memory almost exhausted"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 900, to: 0 }
|
|
datasourceUid: prometheus
|
|
model: { refId: A, expr: "100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [92] } } ] }
|
|
noDataState: OK
|
|
execErrState: Error
|
|
for: 10m
|
|
labels: { severity: warning }
|
|
annotations: { summary: "{{ $labels.host }} memory is {{ $values.A.Value | printf \"%.0f\" }}% used" }
|
|
- uid: container-restarting
|
|
title: "Container restart-looping"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 900, to: 0 }
|
|
datasourceUid: prometheus
|
|
model: { refId: A, expr: "changes(container_start_time_seconds{name!=\"\"}[15m])", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [2] } } ] }
|
|
noDataState: OK
|
|
execErrState: Error
|
|
for: 0m
|
|
labels: { severity: critical }
|
|
annotations: { summary: "{{ $labels.name }} on {{ $labels.host }} restarted {{ $values.A.Value | printf \"%.0f\" }} times in 15 minutes" }
|
|
- uid: postgres-down
|
|
title: "Postgres down"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 900, to: 0 }
|
|
datasourceUid: prometheus
|
|
model: { refId: A, expr: "pg_up", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: lt, params: [1] } } ] }
|
|
noDataState: OK
|
|
execErrState: Error
|
|
for: 1m
|
|
labels: { severity: critical }
|
|
annotations: { summary: "Postgres (storage) is not answering" }
|
|
- uid: backup-stale
|
|
title: "Backup stale"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 900, to: 0 }
|
|
datasourceUid: prometheus
|
|
model: { refId: A, expr: "(time() - restic_backup_last_success_timestamp_seconds) / 3600", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [26] } } ] }
|
|
noDataState: OK
|
|
execErrState: Error
|
|
for: 0m
|
|
labels: { severity: warning }
|
|
annotations: { summary: "restic on {{ $labels.host }}: last good backup {{ $values.A.Value | printf \"%.0f\" }} hours ago" }
|
|
- uid: replica-stale
|
|
title: "Bucket replica stale"
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: { from: 900, to: 0 }
|
|
datasourceUid: prometheus
|
|
model: { refId: A, expr: "(time() - s3_replicate_last_success_timestamp_seconds) / 3600", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [26] } } ] }
|
|
noDataState: OK
|
|
execErrState: Error
|
|
for: 0m
|
|
labels: { severity: warning }
|
|
annotations: { summary: "hel1 -> fsn1 bucket replica: last good run {{ $values.A.Value | printf \"%.0f\" }} hours ago" }
|
|
{% endraw %}
|