ansible: monitoring with Prometheus, Loki and Grafana

Nothing watched the servers: tonight's asxpio crash loop was found by
hand. On asxpio-ops, Prometheus (30 days), Loki (14 days) and Grafana
at gf.asxp.io (WireGuard only), with blackbox_exporter probing every
service from outside and postgres_exporter. On both hosts node_exporter
and cAdvisor on the VPC address, and Alloy shipping container logs, the
journal and Traefik's access log to Loki over the VPC.

Traefik and Forgejo expose their metrics; restic and the bucket replica
record their last success for node_exporter. Dashboards: an overview as
home, plus hosts, containers, Traefik, probes, Postgres and logs.
Alerts by email to me@asxp.io: host or service down, certificate under
14 days, disk or memory nearly full, a container restart-looping,
Postgres down, backup or replica stale.

Assisted-by: Claude:opus-5-5

ansible/roles/monitoring/templates/alert-rules.yaml.j2

@@ -0,0 +1,170 @@
+# Managed by Ansible (role monitoring). Generated; see the commit adding it.
+{% raw %}
+apiVersion: 1
+groups:
+  - orgId: 1
+    name: asxpio
+    folder: Alerts
+    interval: 1m
+    rules:
+      - uid: host-down
+        title: "Host down"
+        condition: C
+        data:
+          - refId: A
+            relativeTimeRange: { from: 900, to: 0 }
+            datasourceUid: prometheus
+            model: { refId: A, expr: "1 - up{job=\"node\"}", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
+          - refId: C
+            datasourceUid: __expr__
+            model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [0] } } ] }
+        noDataState: OK
+        execErrState: Error
+        for: 2m
+        labels: { severity: critical }
+        annotations: { summary: "{{ $labels.host }} stopped answering (node_exporter over the VPC)" }
+      - uid: target-down
+        title: "Scrape target down"
+        condition: C
+        data:
+          - refId: A
+            relativeTimeRange: { from: 900, to: 0 }
+            datasourceUid: prometheus
+            model: { refId: A, expr: "up{job!~\"node|probe_.*\"} == 0", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
+          - refId: C
+            datasourceUid: __expr__
+            model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [-1] } } ] }
+        noDataState: OK
+        execErrState: Error
+        for: 5m
+        labels: { severity: warning }
+        annotations: { summary: "{{ $labels.job }} on {{ $labels.instance }} is not answering metrics scrapes" }
+      - uid: probe-down
+        title: "Service down (outside-in probe)"
+        condition: C
+        data:
+          - refId: A
+            relativeTimeRange: { from: 900, to: 0 }
+            datasourceUid: prometheus
+            model: { refId: A, expr: "1 - probe_success", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
+          - refId: C
+            datasourceUid: __expr__
+            model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [0] } } ] }
+        noDataState: OK
+        execErrState: Error
+        for: 2m
+        labels: { severity: critical }
+        annotations: { summary: "{{ $labels.probe }} ({{ $labels.instance }}) is not reachable" }
+      - uid: cert-expiry
+        title: "Certificate expires soon"
+        condition: C
+        data:
+          - refId: A
+            relativeTimeRange: { from: 900, to: 0 }
+            datasourceUid: prometheus
+            model: { refId: A, expr: "(probe_ssl_earliest_cert_expiry - time()) / 86400", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
+          - refId: C
+            datasourceUid: __expr__
+            model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: lt, params: [14] } } ] }
+        noDataState: OK
+        execErrState: Error
+        for: 1h
+        labels: { severity: warning }
+        annotations: { summary: "{{ $labels.probe }}: certificate expires in {{ $values.A.Value | printf \"%.0f\" }} days" }
+      - uid: disk-full
+        title: "Disk almost full"
+        condition: C
+        data:
+          - refId: A
+            relativeTimeRange: { from: 900, to: 0 }
+            datasourceUid: prometheus
+            model: { refId: A, expr: "100 * (1 - node_filesystem_avail_bytes{fstype!~\"tmpfs|overlay|squashfs|fuse.*\"} / node_filesystem_size_bytes{fstype!~\"tmpfs|overlay|squashfs|fuse.*\"})", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
+          - refId: C
+            datasourceUid: __expr__
+            model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [85] } } ] }
+        noDataState: OK
+        execErrState: Error
+        for: 10m
+        labels: { severity: warning }
+        annotations: { summary: "{{ $labels.host }} {{ $labels.mountpoint }} is {{ $values.A.Value | printf \"%.0f\" }}% full" }
+      - uid: memory-low
+        title: "Memory almost exhausted"
+        condition: C
+        data:
+          - refId: A
+            relativeTimeRange: { from: 900, to: 0 }
+            datasourceUid: prometheus
+            model: { refId: A, expr: "100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
+          - refId: C
+            datasourceUid: __expr__
+            model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [92] } } ] }
+        noDataState: OK
+        execErrState: Error
+        for: 10m
+        labels: { severity: warning }
+        annotations: { summary: "{{ $labels.host }} memory is {{ $values.A.Value | printf \"%.0f\" }}% used" }
+      - uid: container-restarting
+        title: "Container restart-looping"
+        condition: C
+        data:
+          - refId: A
+            relativeTimeRange: { from: 900, to: 0 }
+            datasourceUid: prometheus
+            model: { refId: A, expr: "changes(container_start_time_seconds{name!=\"\"}[15m])", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
+          - refId: C
+            datasourceUid: __expr__
+            model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [2] } } ] }
+        noDataState: OK
+        execErrState: Error
+        for: 0m
+        labels: { severity: critical }
+        annotations: { summary: "{{ $labels.name }} on {{ $labels.host }} restarted {{ $values.A.Value | printf \"%.0f\" }} times in 15 minutes" }
+      - uid: postgres-down
+        title: "Postgres down"
+        condition: C
+        data:
+          - refId: A
+            relativeTimeRange: { from: 900, to: 0 }
+            datasourceUid: prometheus
+            model: { refId: A, expr: "pg_up", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
+          - refId: C
+            datasourceUid: __expr__
+            model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: lt, params: [1] } } ] }
+        noDataState: OK
+        execErrState: Error
+        for: 1m
+        labels: { severity: critical }
+        annotations: { summary: "Postgres (storage) is not answering" }
+      - uid: backup-stale
+        title: "Backup stale"
+        condition: C
+        data:
+          - refId: A
+            relativeTimeRange: { from: 900, to: 0 }
+            datasourceUid: prometheus
+            model: { refId: A, expr: "(time() - restic_backup_last_success_timestamp_seconds) / 3600", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
+          - refId: C
+            datasourceUid: __expr__
+            model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [26] } } ] }
+        noDataState: OK
+        execErrState: Error
+        for: 0m
+        labels: { severity: warning }
+        annotations: { summary: "restic on {{ $labels.host }}: last good backup {{ $values.A.Value | printf \"%.0f\" }} hours ago" }
+      - uid: replica-stale
+        title: "Bucket replica stale"
+        condition: C
+        data:
+          - refId: A
+            relativeTimeRange: { from: 900, to: 0 }
+            datasourceUid: prometheus
+            model: { refId: A, expr: "(time() - s3_replicate_last_success_timestamp_seconds) / 3600", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
+          - refId: C
+            datasourceUid: __expr__
+            model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [26] } } ] }
+        noDataState: OK
+        execErrState: Error
+        for: 0m
+        labels: { severity: warning }
+        annotations: { summary: "hel1 -> fsn1 bucket replica: last good run {{ $values.A.Value | printf \"%.0f\" }} hours ago" }
+{% endraw %}

ansible/roles/monitoring/templates/alerting.yaml.j2

@@ -0,0 +1,17 @@
+apiVersion: 1
+contactPoints:
+  - orgId: 1
+    name: email
+    receivers:
+      - uid: email-me
+        type: email
+        settings:
+          addresses: {{ monitoring_alert_email }}
+          singleEmail: true
+policies:
+  - orgId: 1
+    receiver: email
+    group_by: [alertname, host]
+    group_wait: 1m
+    group_interval: 5m
+    repeat_interval: 4h

ansible/roles/monitoring/templates/blackbox.yml.j2

@@ -0,0 +1,38 @@
+# Managed by Ansible (role monitoring).
+modules:
+  http_2xx:
+    prober: http
+    timeout: 10s
+    http:
+      preferred_ip_protocol: ip4
+      follow_redirects: true
+  # Up means "answers": 401/403/404 are fine (s3 root, allowlisted UIs)
+  http_up:
+    prober: http
+    timeout: 10s
+    http:
+      preferred_ip_protocol: ip4
+      valid_status_codes: [200, 301, 302, 401, 403, 404]
+  tls_connect:
+    prober: tcp
+    timeout: 10s
+    tcp:
+      preferred_ip_protocol: ip4
+      tls: true
+  tcp_connect:
+    prober: tcp
+    timeout: 10s
+    tcp:
+      preferred_ip_protocol: ip4
+  # XMPP client port: stream header, STARTTLS, then the certificate
+  xmpp_starttls:
+    prober: tcp
+    timeout: 10s
+    tcp:
+      preferred_ip_protocol: ip4
+      query_response:
+        - send: "<?xml version='1.0'?><stream:stream to='asxp.io' xmlns='jabber:client' xmlns:stream='http://etherx.jabber.org/streams' version='1.0'>"
+        - expect: "urn:ietf:params:xml:ns:xmpp-tls"
+        - send: "<starttls xmlns='urn:ietf:params:xml:ns:xmpp-tls'/>"
+        - expect: "<proceed"
+        - starttls: true

ansible/roles/monitoring/templates/dashboards.yaml.j2

@@ -0,0 +1,9 @@
+apiVersion: 1
+providers:
+  - name: asxpio
+    type: file
+    disableDeletion: true
+    allowUiUpdates: false
+    options:
+      path: /var/lib/grafana/dashboards
+      foldersFromFilesStructure: true

ansible/roles/monitoring/templates/datasources.yaml.j2

@@ -0,0 +1,14 @@
+apiVersion: 1
+datasources:
+  - name: Prometheus
+    uid: prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    jsonData: { timeInterval: 30s }
+  - name: Loki
+    uid: loki
+    type: loki
+    access: proxy
+    url: http://loki:3100

ansible/roles/monitoring/templates/docker-compose.yml.j2

@@ -0,0 +1,87 @@
+# Managed by Ansible (role monitoring).
+services:
+  prometheus:
+    image: prom/prometheus:{{ monitoring_prometheus_version }}
+    container_name: prometheus
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+      - --storage.tsdb.path=/prometheus
+      - --storage.tsdb.retention.time={{ monitoring_metrics_retention }}
+      - --web.enable-lifecycle
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - prometheus-data:/prometheus
+    networks: [monitoring, traefik, storage]
+    restart: unless-stopped
+    mem_limit: 640m
+
+  loki:
+    image: grafana/loki:{{ monitoring_loki_version }}
+    container_name: loki
+    command: ["-config.file=/etc/loki/loki.yml"]
+    ports:
+      # Alloy on both hosts pushes here over the VPC
+      - "{{ asxpio_ops_vpc_ip }}:{{ monitoring_loki_port }}:3100"
+    volumes:
+      - ./loki.yml:/etc/loki/loki.yml:ro
+      - loki-data:/loki
+    networks: [monitoring]
+    restart: unless-stopped
+    mem_limit: 512m
+
+  grafana:
+    image: grafana/grafana:{{ monitoring_grafana_version }}
+    container_name: grafana
+    env_file: [grafana.env]
+    volumes:
+      - grafana-data:/var/lib/grafana
+      - ./grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./grafana/dashboards:/var/lib/grafana/dashboards:ro
+    networks: [monitoring, traefik]
+    restart: unless-stopped
+    mem_limit: 384m
+    labels:
+      - traefik.enable=true
+      - traefik.docker.network=traefik
+      - traefik.http.routers.grafana.rule=Host(`{{ monitoring_grafana_host }}`)
+      - traefik.http.routers.grafana.entrypoints=websecure
+      - traefik.http.routers.grafana.tls=true
+      - traefik.http.routers.grafana.tls.certresolver=letsencrypt
+      - traefik.http.routers.grafana.middlewares=grafana-wireguard-only
+      - traefik.http.middlewares.grafana-wireguard-only.ipallowlist.sourcerange={{ wireguard_network_cidr }}
+      - traefik.http.services.grafana.loadbalancer.server.port=3000
+
+  blackbox:
+    image: prom/blackbox-exporter:{{ monitoring_blackbox_version }}
+    container_name: blackbox
+    command: ["--config.file=/etc/blackbox/blackbox.yml"]
+    volumes:
+      - ./blackbox.yml:/etc/blackbox/blackbox.yml:ro
+    networks: [monitoring]
+    restart: unless-stopped
+    mem_limit: 64m
+
+  postgres-exporter:
+    image: quay.io/prometheuscommunity/postgres-exporter:{{ monitoring_postgres_exporter_version }}
+    container_name: postgres-exporter
+    # The password comes from the storage stack's own .env (same host)
+    env_file: [{{ services_base_path }}/storage/.env]
+    environment:
+      DATA_SOURCE_URI: storage-postgres:5432/postgres?sslmode=disable
+      DATA_SOURCE_USER: postgres
+    entrypoint: ["/bin/sh", "-c", "DATA_SOURCE_PASS=\"$$POSTGRES_PASSWORD\" exec /bin/postgres_exporter"]
+    networks: [monitoring, storage]
+    restart: unless-stopped
+    mem_limit: 64m
+
+volumes:
+  prometheus-data:
+  loki-data:
+  grafana-data:
+
+networks:
+  monitoring:
+  traefik:
+    external: true
+  storage:
+    external: true

ansible/roles/monitoring/templates/grafana.env.j2

@@ -0,0 +1,17 @@
+GF_SERVER_ROOT_URL=https://{{ monitoring_grafana_host }}/
+GF_SECURITY_ADMIN_USER=admin
+GF_SECURITY_ADMIN_PASSWORD={{ monitoring_grafana_admin_password }}
+GF_SECURITY_COOKIE_SECURE=true
+GF_USERS_ALLOW_SIGN_UP=false
+GF_AUTH_ANONYMOUS_ENABLED=false
+GF_ANALYTICS_REPORTING_ENABLED=false
+GF_ANALYTICS_CHECK_FOR_UPDATES=false
+GF_NEWS_NEWS_FEED_ENABLED=false
+GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/var/lib/grafana/dashboards/overview.json
+GF_SMTP_ENABLED=true
+GF_SMTP_HOST=smtp.fastmail.com:587
+GF_SMTP_USER={{ vault_forgejo_smtp_user }}
+GF_SMTP_PASSWORD={{ vault_forgejo_smtp_password }}
+GF_SMTP_FROM_ADDRESS={{ vault_forgejo_smtp_user }}
+GF_SMTP_FROM_NAME=asxpio monitoring
+GF_SMTP_STARTTLS_POLICY=MandatoryStartTLS

ansible/roles/monitoring/templates/loki.yml.j2

@@ -0,0 +1,37 @@
+# Managed by Ansible (role monitoring).
+auth_enabled: false
+server:
+  http_listen_port: 3100
+  log_level: warn
+common:
+  path_prefix: /loki
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+  storage:
+    filesystem:
+      chunks_directory: /loki/chunks
+      rules_directory: /loki/rules
+schema_config:
+  configs:
+    - from: "2026-11-01"
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+limits_config:
+  retention_period: {{ monitoring_logs_retention }}
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+  ingestion_rate_mb: 8
+  ingestion_burst_size_mb: 16
+  volume_enabled: true
+compactor:
+  working_directory: /loki/compactor
+  retention_enabled: true
+  delete_request_store: filesystem
+analytics:
+  reporting_enabled: false

ansible/roles/monitoring/templates/prometheus.yml.j2

@@ -0,0 +1,75 @@
+# Managed by Ansible (role monitoring).
+global:
+  scrape_interval: 30s
+  evaluation_interval: 30s
+
+scrape_configs:
+  - job_name: node
+    static_configs:
+{% for h in monitoring_hosts %}
+      - targets: ["{{ h.address }}:{{ monitoring_node_exporter_port }}"]
+        labels: { host: {{ h.name }} }
+{% endfor %}
+
+  - job_name: cadvisor
+    scrape_interval: 60s
+    static_configs:
+{% for h in monitoring_hosts %}
+      - targets: ["{{ h.address }}:{{ monitoring_cadvisor_port }}"]
+        labels: { host: {{ h.name }} }
+{% endfor %}
+    metric_relabel_configs:
+      # Short-lived CI job containers would bloat the series count
+      - source_labels: [name]
+        regex: FORGEJO-ACTIONS-TASK-.*
+        action: drop
+
+  - job_name: alloy
+    static_configs:
+{% for h in monitoring_hosts %}
+      - targets: ["{{ h.address }}:12345"]
+        labels: { host: {{ h.name }} }
+{% endfor %}
+
+  - job_name: traefik
+    static_configs:
+      - targets: ["traefik:8082"]
+        labels: { host: asxpio-ops }
+
+  - job_name: forgejo
+    metrics_path: /metrics
+    authorization:
+      credentials: "{{ monitoring_forgejo_metrics_token }}"
+    static_configs:
+      - targets: ["forgejo:3000"]
+        labels: { host: asxpio-ops }
+
+  - job_name: postgres
+    static_configs:
+      - targets: ["postgres-exporter:9187"]
+        labels: { host: asxpio-ops }
+
+  - job_name: monitoring
+    static_configs:
+      - targets: ["localhost:9090", "loki:3100", "grafana:3000", "blackbox:9115"]
+        labels: { host: asxpio-ops }
+
+{% for module in monitoring_probes | map(attribute='module') | unique %}
+  - job_name: probe_{{ module }}
+    metrics_path: /probe
+    params:
+      module: [{{ module }}]
+    static_configs:
+{% for p in monitoring_probes if p.module == module %}
+      - targets: ["{{ p.target }}"]
+        labels: { probe: "{{ p.name }}" }
+{% endfor %}
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: blackbox:9115
+
+{% endfor %}