Sergei Poljanski
2359370e24
All checks were successful
deploy / ansible (push) Successful in 7m59s
Each SSH login from Ansible and CI deploys creates its own session-N.scope unit. As a journal label that made 3400+ streams and Loki rejected new ones with 429, dropping container logs. Map them all to session.scope. cAdvisor's image healthcheck probes localhost:8080, but it listens on the VPC address on 9180, so both containers showed unhealthy. Assisted-by: Claude:opus-5-5
86 lines
2.3 KiB
Django/Jinja
86 lines
2.3 KiB
Django/Jinja
// Managed by Ansible (role monitoring_agent): logs of {{ inventory_hostname }} to Loki.
|
|
|
|
discovery.docker "containers" {
|
|
host = "unix:///var/run/docker.sock"
|
|
}
|
|
|
|
discovery.relabel "containers" {
|
|
targets = discovery.docker.containers.targets
|
|
|
|
rule {
|
|
source_labels = ["__meta_docker_container_name"]
|
|
regex = "/(.*)"
|
|
target_label = "container"
|
|
}
|
|
rule {
|
|
source_labels = ["__meta_docker_container_label_com_docker_compose_project"]
|
|
target_label = "compose_project"
|
|
}
|
|
rule {
|
|
source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
|
|
target_label = "compose_service"
|
|
}
|
|
}
|
|
|
|
loki.source.docker "containers" {
|
|
host = "unix:///var/run/docker.sock"
|
|
targets = discovery.docker.containers.targets
|
|
relabel_rules = discovery.relabel.containers.rules
|
|
labels = { job = "docker" }
|
|
forward_to = [loki.process.containers.receiver]
|
|
}
|
|
|
|
loki.process "containers" {
|
|
// CI job containers are many, short-lived and noisy: the runner logs suffice
|
|
stage.match {
|
|
selector = "{container=~\"FORGEJO-ACTIONS-TASK-.*\"}"
|
|
action = "drop"
|
|
}
|
|
forward_to = [loki.write.default.receiver]
|
|
}
|
|
|
|
loki.relabel "journal" {
|
|
forward_to = []
|
|
|
|
rule {
|
|
source_labels = ["__journal__systemd_unit"]
|
|
target_label = "unit"
|
|
}
|
|
// Every SSH login (Ansible, CI deploys) gets its own session-N.scope:
|
|
// one stream each blew Loki's active stream limit.
|
|
rule {
|
|
source_labels = ["unit"]
|
|
regex = "session-\\d+\\.scope"
|
|
target_label = "unit"
|
|
replacement = "session.scope"
|
|
}
|
|
rule {
|
|
source_labels = ["__journal_priority_keyword"]
|
|
target_label = "level"
|
|
}
|
|
}
|
|
|
|
loki.source.journal "journal" {
|
|
max_age = "12h"
|
|
relabel_rules = loki.relabel.journal.rules
|
|
labels = { job = "journal" }
|
|
forward_to = [loki.write.default.receiver]
|
|
}
|
|
{% if inventory_hostname == 'asxpio-ops' %}
|
|
|
|
local.file_match "traefik" {
|
|
path_targets = [{ "__path__" = "/traefik-logs/access.log", job = "traefik-access" }]
|
|
}
|
|
|
|
loki.source.file "traefik" {
|
|
targets = local.file_match.traefik.targets
|
|
forward_to = [loki.write.default.receiver]
|
|
}
|
|
{% endif %}
|
|
|
|
loki.write "default" {
|
|
endpoint {
|
|
url = "http://{{ asxpio_ops_vpc_ip }}:{{ monitoring_loki_port }}/loki/api/v1/push"
|
|
}
|
|
external_labels = { host = "{{ inventory_hostname }}" }
|
|
}
|