Sergei Poljanski
e20f5a7d07
All checks were successful
deploy / ansible (push) Successful in 7m23s
Nothing watched the servers: tonight's asxpio crash loop was found by hand. On asxpio-ops, Prometheus (30 days), Loki (14 days) and Grafana at gf.asxp.io (WireGuard only), with blackbox_exporter probing every service from outside and postgres_exporter. On both hosts node_exporter and cAdvisor on the VPC address, and Alloy shipping container logs, the journal and Traefik's access log to Loki over the VPC. Traefik and Forgejo expose their metrics; restic and the bucket replica record their last success for node_exporter. Dashboards: an overview as home, plus hosts, containers, Traefik, probes, Postgres and logs. Alerts by email to me@asxp.io: host or service down, certificate under 14 days, disk or memory nearly full, a container restart-looping, Postgres down, backup or replica stale. Assisted-by: Claude:opus-5-5
87 lines
2.9 KiB
Django/Jinja
87 lines
2.9 KiB
Django/Jinja
# Managed by Ansible (role monitoring).
|
|
services:
|
|
prometheus:
|
|
image: prom/prometheus:{{ monitoring_prometheus_version }}
|
|
container_name: prometheus
|
|
command:
|
|
- --config.file=/etc/prometheus/prometheus.yml
|
|
- --storage.tsdb.path=/prometheus
|
|
- --storage.tsdb.retention.time={{ monitoring_metrics_retention }}
|
|
- --web.enable-lifecycle
|
|
volumes:
|
|
- ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
|
- prometheus-data:/prometheus
|
|
networks: [monitoring, traefik, storage]
|
|
restart: unless-stopped
|
|
mem_limit: 640m
|
|
|
|
loki:
|
|
image: grafana/loki:{{ monitoring_loki_version }}
|
|
container_name: loki
|
|
command: ["-config.file=/etc/loki/loki.yml"]
|
|
ports:
|
|
# Alloy on both hosts pushes here over the VPC
|
|
- "{{ asxpio_ops_vpc_ip }}:{{ monitoring_loki_port }}:3100"
|
|
volumes:
|
|
- ./loki.yml:/etc/loki/loki.yml:ro
|
|
- loki-data:/loki
|
|
networks: [monitoring]
|
|
restart: unless-stopped
|
|
mem_limit: 512m
|
|
|
|
grafana:
|
|
image: grafana/grafana:{{ monitoring_grafana_version }}
|
|
container_name: grafana
|
|
env_file: [grafana.env]
|
|
volumes:
|
|
- grafana-data:/var/lib/grafana
|
|
- ./grafana/provisioning:/etc/grafana/provisioning:ro
|
|
- ./grafana/dashboards:/var/lib/grafana/dashboards:ro
|
|
networks: [monitoring, traefik]
|
|
restart: unless-stopped
|
|
mem_limit: 384m
|
|
labels:
|
|
- traefik.enable=true
|
|
- traefik.docker.network=traefik
|
|
- traefik.http.routers.grafana.rule=Host(`{{ monitoring_grafana_host }}`)
|
|
- traefik.http.routers.grafana.entrypoints=websecure
|
|
- traefik.http.routers.grafana.tls=true
|
|
- traefik.http.routers.grafana.tls.certresolver=letsencrypt
|
|
- traefik.http.routers.grafana.middlewares=grafana-wireguard-only
|
|
- traefik.http.middlewares.grafana-wireguard-only.ipallowlist.sourcerange={{ wireguard_network_cidr }}
|
|
- traefik.http.services.grafana.loadbalancer.server.port=3000
|
|
|
|
blackbox:
|
|
image: prom/blackbox-exporter:{{ monitoring_blackbox_version }}
|
|
container_name: blackbox
|
|
command: ["--config.file=/etc/blackbox/blackbox.yml"]
|
|
volumes:
|
|
- ./blackbox.yml:/etc/blackbox/blackbox.yml:ro
|
|
networks: [monitoring]
|
|
restart: unless-stopped
|
|
mem_limit: 64m
|
|
|
|
postgres-exporter:
|
|
image: quay.io/prometheuscommunity/postgres-exporter:{{ monitoring_postgres_exporter_version }}
|
|
container_name: postgres-exporter
|
|
# The password comes from the storage stack's own .env (same host)
|
|
env_file: [{{ services_base_path }}/storage/.env]
|
|
environment:
|
|
DATA_SOURCE_URI: storage-postgres:5432/postgres?sslmode=disable
|
|
DATA_SOURCE_USER: postgres
|
|
entrypoint: ["/bin/sh", "-c", "DATA_SOURCE_PASS=\"$$POSTGRES_PASSWORD\" exec /bin/postgres_exporter"]
|
|
networks: [monitoring, storage]
|
|
restart: unless-stopped
|
|
mem_limit: 64m
|
|
|
|
volumes:
|
|
prometheus-data:
|
|
loki-data:
|
|
grafana-data:
|
|
|
|
networks:
|
|
monitoring:
|
|
traefik:
|
|
external: true
|
|
storage:
|
|
external: true
|