Sergei Poljanski
e20f5a7d07
All checks were successful
deploy / ansible (push) Successful in 7m23s
Nothing watched the servers: tonight's asxpio crash loop was found by hand. On asxpio-ops, Prometheus (30 days), Loki (14 days) and Grafana at gf.asxp.io (WireGuard only), with blackbox_exporter probing every service from outside and postgres_exporter. On both hosts node_exporter and cAdvisor on the VPC address, and Alloy shipping container logs, the journal and Traefik's access log to Loki over the VPC. Traefik and Forgejo expose their metrics; restic and the bucket replica record their last success for node_exporter. Dashboards: an overview as home, plus hosts, containers, Traefik, probes, Postgres and logs. Alerts by email to me@asxp.io: host or service down, certificate under 14 days, disk or memory nearly full, a container restart-looping, Postgres down, backup or replica stale. Assisted-by: Claude:opus-5-5
75 lines
2 KiB
Django/Jinja
75 lines
2 KiB
Django/Jinja
# Managed by Ansible (role monitoring).
|
|
global:
|
|
scrape_interval: 30s
|
|
evaluation_interval: 30s
|
|
|
|
scrape_configs:
|
|
- job_name: node
|
|
static_configs:
|
|
{% for h in monitoring_hosts %}
|
|
- targets: ["{{ h.address }}:{{ monitoring_node_exporter_port }}"]
|
|
labels: { host: {{ h.name }} }
|
|
{% endfor %}
|
|
|
|
- job_name: cadvisor
|
|
scrape_interval: 60s
|
|
static_configs:
|
|
{% for h in monitoring_hosts %}
|
|
- targets: ["{{ h.address }}:{{ monitoring_cadvisor_port }}"]
|
|
labels: { host: {{ h.name }} }
|
|
{% endfor %}
|
|
metric_relabel_configs:
|
|
# Short-lived CI job containers would bloat the series count
|
|
- source_labels: [name]
|
|
regex: FORGEJO-ACTIONS-TASK-.*
|
|
action: drop
|
|
|
|
- job_name: alloy
|
|
static_configs:
|
|
{% for h in monitoring_hosts %}
|
|
- targets: ["{{ h.address }}:12345"]
|
|
labels: { host: {{ h.name }} }
|
|
{% endfor %}
|
|
|
|
- job_name: traefik
|
|
static_configs:
|
|
- targets: ["traefik:8082"]
|
|
labels: { host: asxpio-ops }
|
|
|
|
- job_name: forgejo
|
|
metrics_path: /metrics
|
|
authorization:
|
|
credentials: "{{ monitoring_forgejo_metrics_token }}"
|
|
static_configs:
|
|
- targets: ["forgejo:3000"]
|
|
labels: { host: asxpio-ops }
|
|
|
|
- job_name: postgres
|
|
static_configs:
|
|
- targets: ["postgres-exporter:9187"]
|
|
labels: { host: asxpio-ops }
|
|
|
|
- job_name: monitoring
|
|
static_configs:
|
|
- targets: ["localhost:9090", "loki:3100", "grafana:3000", "blackbox:9115"]
|
|
labels: { host: asxpio-ops }
|
|
|
|
{% for module in monitoring_probes | map(attribute='module') | unique %}
|
|
- job_name: probe_{{ module }}
|
|
metrics_path: /probe
|
|
params:
|
|
module: [{{ module }}]
|
|
static_configs:
|
|
{% for p in monitoring_probes if p.module == module %}
|
|
- targets: ["{{ p.target }}"]
|
|
labels: { probe: "{{ p.name }}" }
|
|
{% endfor %}
|
|
relabel_configs:
|
|
- source_labels: [__address__]
|
|
target_label: __param_target
|
|
- source_labels: [__param_target]
|
|
target_label: instance
|
|
- target_label: __address__
|
|
replacement: blackbox:9115
|
|
|
|
{% endfor %}
|