ansible: monitoring with Prometheus, Loki and Grafana

Nothing watched the servers: tonight's asxpio crash loop was found by
hand. On asxpio-ops, Prometheus (30 days), Loki (14 days) and Grafana
at gf.asxp.io (WireGuard only), with blackbox_exporter probing every
service from outside and postgres_exporter. On both hosts node_exporter
and cAdvisor on the VPC address, and Alloy shipping container logs, the
journal and Traefik's access log to Loki over the VPC.

Traefik and Forgejo expose their metrics; restic and the bucket replica
record their last success for node_exporter. Dashboards: an overview as
home, plus hosts, containers, Traefik, probes, Postgres and logs.
Alerts by email to me@asxp.io: host or service down, certificate under
14 days, disk or memory nearly full, a container restart-looping,
Postgres down, backup or replica stale.

Assisted-by: Claude:opus-5-5

ansible/roles/monitoring_agent/templates/config.alloy.j2

@@ -0,0 +1,78 @@
+// Managed by Ansible (role monitoring_agent): logs of {{ inventory_hostname }} to Loki.
+
+discovery.docker "containers" {
+  host = "unix:///var/run/docker.sock"
+}
+
+discovery.relabel "containers" {
+  targets = discovery.docker.containers.targets
+
+  rule {
+    source_labels = ["__meta_docker_container_name"]
+    regex         = "/(.*)"
+    target_label  = "container"
+  }
+  rule {
+    source_labels = ["__meta_docker_container_label_com_docker_compose_project"]
+    target_label  = "compose_project"
+  }
+  rule {
+    source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
+    target_label  = "compose_service"
+  }
+}
+
+loki.source.docker "containers" {
+  host          = "unix:///var/run/docker.sock"
+  targets       = discovery.docker.containers.targets
+  relabel_rules = discovery.relabel.containers.rules
+  labels        = { job = "docker" }
+  forward_to    = [loki.process.containers.receiver]
+}
+
+loki.process "containers" {
+  // CI job containers are many, short-lived and noisy: the runner logs suffice
+  stage.match {
+    selector = "{container=~\"FORGEJO-ACTIONS-TASK-.*\"}"
+    action   = "drop"
+  }
+  forward_to = [loki.write.default.receiver]
+}
+
+loki.relabel "journal" {
+  forward_to = []
+
+  rule {
+    source_labels = ["__journal__systemd_unit"]
+    target_label  = "unit"
+  }
+  rule {
+    source_labels = ["__journal_priority_keyword"]
+    target_label  = "level"
+  }
+}
+
+loki.source.journal "journal" {
+  max_age       = "12h"
+  relabel_rules = loki.relabel.journal.rules
+  labels        = { job = "journal" }
+  forward_to    = [loki.write.default.receiver]
+}
+{% if inventory_hostname == 'asxpio-ops' %}
+
+local.file_match "traefik" {
+  path_targets = [{ "__path__" = "/traefik-logs/access.log", job = "traefik-access" }]
+}
+
+loki.source.file "traefik" {
+  targets    = local.file_match.traefik.targets
+  forward_to = [loki.write.default.receiver]
+}
+{% endif %}
+
+loki.write "default" {
+  endpoint {
+    url = "http://{{ asxpio_ops_vpc_ip }}:{{ monitoring_loki_port }}/loki/api/v1/push"
+  }
+  external_labels = { host = "{{ inventory_hostname }}" }
+}

ansible/roles/monitoring_agent/templates/docker-compose.yml.j2

@@ -0,0 +1,66 @@
+# Managed by Ansible (role monitoring_agent).
+services:
+  node-exporter:
+    image: prom/node-exporter:{{ monitoring_node_exporter_version }}
+    container_name: node-exporter
+    command:
+      - --path.rootfs=/host
+      - --web.listen-address={{ vpc_ip }}:{{ monitoring_node_exporter_port }}
+      - --collector.textfile.directory=/textfile
+      - --collector.filesystem.mount-points-exclude=^/(dev|proc|run|sys|var/lib/docker/.+)($|/)
+    network_mode: host
+    pid: host
+    volumes:
+      - /:/host:ro,rslave
+      - {{ monitoring_textfile_dir }}:/textfile:ro
+    restart: unless-stopped
+    mem_limit: 64m
+
+  cadvisor:
+    image: ghcr.io/google/cadvisor:{{ monitoring_cadvisor_version }}
+    container_name: cadvisor
+    command:
+      - --listen_ip={{ vpc_ip }}
+      - --port={{ monitoring_cadvisor_port }}
+      - --docker_only=true
+      - --housekeeping_interval=30s
+      - --store_container_labels=false
+      - --whitelisted_container_labels=com.docker.compose.project,com.docker.compose.service
+      - --disable_metrics=percpu,sched,tcp,udp,hugetlb,referenced_memory,cpu_topology,resctrl,advtcp,process
+    network_mode: host
+    privileged: true
+    devices:
+      - /dev/kmsg
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:ro
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+      - /dev/disk/:/dev/disk:ro
+    restart: unless-stopped
+    mem_limit: 192m
+
+  alloy:
+    image: grafana/alloy:{{ monitoring_alloy_version }}
+    container_name: alloy
+    command:
+      - run
+      - --server.http.listen-addr={{ vpc_ip }}:12345
+      - --storage.path=/var/lib/alloy/data
+      - /etc/alloy/config.alloy
+    network_mode: host
+    volumes:
+      - ./config.alloy:/etc/alloy/config.alloy:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - /var/log/journal:/var/log/journal:ro
+      - /run/log/journal:/run/log/journal:ro
+      - /etc/machine-id:/etc/machine-id:ro
+{% if inventory_hostname == 'asxpio-ops' %}
+      - /opt/traefik/logs:/traefik-logs:ro
+{% endif %}
+      - alloy-data:/var/lib/alloy/data
+    restart: unless-stopped
+    mem_limit: 256m
+
+volumes:
+  alloy-data: