monitoring: collapse session scopes, fix cadvisor healthcheck
All checks were successful
deploy / ansible (push) Successful in 7m59s
All checks were successful
deploy / ansible (push) Successful in 7m59s
Each SSH login from Ansible and CI deploys creates its own session-N.scope unit. As a journal label that made 3400+ streams and Loki rejected new ones with 429, dropping container logs. Map them all to session.scope. cAdvisor's image healthcheck probes localhost:8080, but it listens on the VPC address on 9180, so both containers showed unhealthy. Assisted-by: Claude:opus-5-5
This commit is contained in:
parent
85603657c6
commit
2359370e24
2 changed files with 14 additions and 0 deletions
|
|
@ -46,6 +46,14 @@ loki.relabel "journal" {
|
||||||
source_labels = ["__journal__systemd_unit"]
|
source_labels = ["__journal__systemd_unit"]
|
||||||
target_label = "unit"
|
target_label = "unit"
|
||||||
}
|
}
|
||||||
|
// Every SSH login (Ansible, CI deploys) gets its own session-N.scope:
|
||||||
|
// one stream each blew Loki's active stream limit.
|
||||||
|
rule {
|
||||||
|
source_labels = ["unit"]
|
||||||
|
regex = "session-\\d+\\.scope"
|
||||||
|
target_label = "unit"
|
||||||
|
replacement = "session.scope"
|
||||||
|
}
|
||||||
rule {
|
rule {
|
||||||
source_labels = ["__journal_priority_keyword"]
|
source_labels = ["__journal_priority_keyword"]
|
||||||
target_label = "level"
|
target_label = "level"
|
||||||
|
|
|
||||||
|
|
@ -37,6 +37,12 @@ services:
|
||||||
- /sys:/sys:ro
|
- /sys:/sys:ro
|
||||||
- /var/lib/docker/:/var/lib/docker:ro
|
- /var/lib/docker/:/var/lib/docker:ro
|
||||||
- /dev/disk/:/dev/disk:ro
|
- /dev/disk/:/dev/disk:ro
|
||||||
|
healthcheck:
|
||||||
|
# The image probes localhost:8080; we listen on the VPC address only.
|
||||||
|
test: ["CMD", "wget", "-q", "--tries=1", "--spider", "http://{{ vpc_ip }}:{{ monitoring_cadvisor_port }}/healthz"]
|
||||||
|
interval: 30s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 3
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
mem_limit: 192m
|
mem_limit: 192m
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue