ansible: monitoring with Prometheus, Loki and Grafana
All checks were successful
deploy / ansible (push) Successful in 7m23s
All checks were successful
deploy / ansible (push) Successful in 7m23s
Nothing watched the servers: tonight's asxpio crash loop was found by hand. On asxpio-ops, Prometheus (30 days), Loki (14 days) and Grafana at gf.asxp.io (WireGuard only), with blackbox_exporter probing every service from outside and postgres_exporter. On both hosts node_exporter and cAdvisor on the VPC address, and Alloy shipping container logs, the journal and Traefik's access log to Loki over the VPC. Traefik and Forgejo expose their metrics; restic and the bucket replica record their last success for node_exporter. Dashboards: an overview as home, plus hosts, containers, Traefik, probes, Postgres and logs. Alerts by email to me@asxp.io: host or service down, certificate under 14 days, disk or memory nearly full, a container restart-looping, Postgres down, backup or replica stale. Assisted-by: Claude:opus-5-5
This commit is contained in:
parent
5289542dca
commit
e20f5a7d07
GPG key ID:
4F8851660FA4121B
28 changed files with 24705 additions and 161 deletions
170
ansible/roles/monitoring/templates/alert-rules.yaml.j2
Normal file
170
ansible/roles/monitoring/templates/alert-rules.yaml.j2
Normal file
|
|
@ -0,0 +1,170 @@
|
|||
# Managed by Ansible (role monitoring). Generated; see the commit adding it.
|
||||
{% raw %}
|
||||
apiVersion: 1
|
||||
groups:
|
||||
- orgId: 1
|
||||
name: asxpio
|
||||
folder: Alerts
|
||||
interval: 1m
|
||||
rules:
|
||||
- uid: host-down
|
||||
title: "Host down"
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 900, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model: { refId: A, expr: "1 - up{job=\"node\"}", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [0] } } ] }
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 2m
|
||||
labels: { severity: critical }
|
||||
annotations: { summary: "{{ $labels.host }} stopped answering (node_exporter over the VPC)" }
|
||||
- uid: target-down
|
||||
title: "Scrape target down"
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 900, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model: { refId: A, expr: "up{job!~\"node|probe_.*\"} == 0", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [-1] } } ] }
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 5m
|
||||
labels: { severity: warning }
|
||||
annotations: { summary: "{{ $labels.job }} on {{ $labels.instance }} is not answering metrics scrapes" }
|
||||
- uid: probe-down
|
||||
title: "Service down (outside-in probe)"
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 900, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model: { refId: A, expr: "1 - probe_success", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [0] } } ] }
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 2m
|
||||
labels: { severity: critical }
|
||||
annotations: { summary: "{{ $labels.probe }} ({{ $labels.instance }}) is not reachable" }
|
||||
- uid: cert-expiry
|
||||
title: "Certificate expires soon"
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 900, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model: { refId: A, expr: "(probe_ssl_earliest_cert_expiry - time()) / 86400", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: lt, params: [14] } } ] }
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 1h
|
||||
labels: { severity: warning }
|
||||
annotations: { summary: "{{ $labels.probe }}: certificate expires in {{ $values.A.Value | printf \"%.0f\" }} days" }
|
||||
- uid: disk-full
|
||||
title: "Disk almost full"
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 900, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model: { refId: A, expr: "100 * (1 - node_filesystem_avail_bytes{fstype!~\"tmpfs|overlay|squashfs|fuse.*\"} / node_filesystem_size_bytes{fstype!~\"tmpfs|overlay|squashfs|fuse.*\"})", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [85] } } ] }
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 10m
|
||||
labels: { severity: warning }
|
||||
annotations: { summary: "{{ $labels.host }} {{ $labels.mountpoint }} is {{ $values.A.Value | printf \"%.0f\" }}% full" }
|
||||
- uid: memory-low
|
||||
title: "Memory almost exhausted"
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 900, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model: { refId: A, expr: "100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [92] } } ] }
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 10m
|
||||
labels: { severity: warning }
|
||||
annotations: { summary: "{{ $labels.host }} memory is {{ $values.A.Value | printf \"%.0f\" }}% used" }
|
||||
- uid: container-restarting
|
||||
title: "Container restart-looping"
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 900, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model: { refId: A, expr: "changes(container_start_time_seconds{name!=\"\"}[15m])", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [2] } } ] }
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 0m
|
||||
labels: { severity: critical }
|
||||
annotations: { summary: "{{ $labels.name }} on {{ $labels.host }} restarted {{ $values.A.Value | printf \"%.0f\" }} times in 15 minutes" }
|
||||
- uid: postgres-down
|
||||
title: "Postgres down"
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 900, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model: { refId: A, expr: "pg_up", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: lt, params: [1] } } ] }
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 1m
|
||||
labels: { severity: critical }
|
||||
annotations: { summary: "Postgres (storage) is not answering" }
|
||||
- uid: backup-stale
|
||||
title: "Backup stale"
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 900, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model: { refId: A, expr: "(time() - restic_backup_last_success_timestamp_seconds) / 3600", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [26] } } ] }
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 0m
|
||||
labels: { severity: warning }
|
||||
annotations: { summary: "restic on {{ $labels.host }}: last good backup {{ $values.A.Value | printf \"%.0f\" }} hours ago" }
|
||||
- uid: replica-stale
|
||||
title: "Bucket replica stale"
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 900, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model: { refId: A, expr: "(time() - s3_replicate_last_success_timestamp_seconds) / 3600", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [26] } } ] }
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 0m
|
||||
labels: { severity: warning }
|
||||
annotations: { summary: "hel1 -> fsn1 bucket replica: last good run {{ $values.A.Value | printf \"%.0f\" }} hours ago" }
|
||||
{% endraw %}
|
||||
Loading…
Add table
Add a link
Reference in a new issue