Sergei Poljanski
e20f5a7d07
All checks were successful
deploy / ansible (push) Successful in 7m23s
Nothing watched the servers: tonight's asxpio crash loop was found by hand. On asxpio-ops, Prometheus (30 days), Loki (14 days) and Grafana at gf.asxp.io (WireGuard only), with blackbox_exporter probing every service from outside and postgres_exporter. On both hosts node_exporter and cAdvisor on the VPC address, and Alloy shipping container logs, the journal and Traefik's access log to Loki over the VPC. Traefik and Forgejo expose their metrics; restic and the bucket replica record their last success for node_exporter. Dashboards: an overview as home, plus hosts, containers, Traefik, probes, Postgres and logs. Alerts by email to me@asxp.io: host or service down, certificate under 14 days, disk or memory nearly full, a container restart-looping, Postgres down, backup or replica stale. Assisted-by: Claude:opus-5-5
78 lines
2 KiB
Django/Jinja
78 lines
2 KiB
Django/Jinja
// Managed by Ansible (role monitoring_agent): logs of {{ inventory_hostname }} to Loki.
|
|
|
|
discovery.docker "containers" {
|
|
host = "unix:///var/run/docker.sock"
|
|
}
|
|
|
|
discovery.relabel "containers" {
|
|
targets = discovery.docker.containers.targets
|
|
|
|
rule {
|
|
source_labels = ["__meta_docker_container_name"]
|
|
regex = "/(.*)"
|
|
target_label = "container"
|
|
}
|
|
rule {
|
|
source_labels = ["__meta_docker_container_label_com_docker_compose_project"]
|
|
target_label = "compose_project"
|
|
}
|
|
rule {
|
|
source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
|
|
target_label = "compose_service"
|
|
}
|
|
}
|
|
|
|
loki.source.docker "containers" {
|
|
host = "unix:///var/run/docker.sock"
|
|
targets = discovery.docker.containers.targets
|
|
relabel_rules = discovery.relabel.containers.rules
|
|
labels = { job = "docker" }
|
|
forward_to = [loki.process.containers.receiver]
|
|
}
|
|
|
|
loki.process "containers" {
|
|
// CI job containers are many, short-lived and noisy: the runner logs suffice
|
|
stage.match {
|
|
selector = "{container=~\"FORGEJO-ACTIONS-TASK-.*\"}"
|
|
action = "drop"
|
|
}
|
|
forward_to = [loki.write.default.receiver]
|
|
}
|
|
|
|
loki.relabel "journal" {
|
|
forward_to = []
|
|
|
|
rule {
|
|
source_labels = ["__journal__systemd_unit"]
|
|
target_label = "unit"
|
|
}
|
|
rule {
|
|
source_labels = ["__journal_priority_keyword"]
|
|
target_label = "level"
|
|
}
|
|
}
|
|
|
|
loki.source.journal "journal" {
|
|
max_age = "12h"
|
|
relabel_rules = loki.relabel.journal.rules
|
|
labels = { job = "journal" }
|
|
forward_to = [loki.write.default.receiver]
|
|
}
|
|
{% if inventory_hostname == 'asxpio-ops' %}
|
|
|
|
local.file_match "traefik" {
|
|
path_targets = [{ "__path__" = "/traefik-logs/access.log", job = "traefik-access" }]
|
|
}
|
|
|
|
loki.source.file "traefik" {
|
|
targets = local.file_match.traefik.targets
|
|
forward_to = [loki.write.default.receiver]
|
|
}
|
|
{% endif %}
|
|
|
|
loki.write "default" {
|
|
endpoint {
|
|
url = "http://{{ asxpio_ops_vpc_ip }}:{{ monitoring_loki_port }}/loki/api/v1/push"
|
|
}
|
|
external_labels = { host = "{{ inventory_hostname }}" }
|
|
}
|