From 2359370e245e05b51c2ab06206e4a4eab4e63edc Mon Sep 17 00:00:00 2001
From: Sergei Poljanski <me@asxp.io>
Date: Fri, 13 Nov 2026 06:42:10 +0400
Subject: [PATCH] monitoring: collapse session scopes, fix cadvisor healthcheck

Each SSH login from Ansible and CI deploys creates its own
session-N.scope unit. As a journal label that made 3400+ streams and
Loki rejected new ones with 429, dropping container logs. Map them all
to session.scope.

cAdvisor's image healthcheck probes localhost:8080, but it listens on
the VPC address on 9180, so both containers showed unhealthy.

Assisted-by: Claude:opus-5-5
---
 ansible/roles/monitoring_agent/templates/config.alloy.j2  | 8 ++++++++
 .../monitoring_agent/templates/docker-compose.yml.j2      | 6 ++++++
 2 files changed, 14 insertions(+)

diff --git a/ansible/roles/monitoring_agent/templates/config.alloy.j2 b/ansible/roles/monitoring_agent/templates/config.alloy.j2
index 719f95b..8ed946b 100644
--- a/ansible/roles/monitoring_agent/templates/config.alloy.j2
+++ b/ansible/roles/monitoring_agent/templates/config.alloy.j2
@@ -46,6 +46,14 @@ loki.relabel "journal" {
     source_labels = ["__journal__systemd_unit"]
     target_label  = "unit"
   }
+  // Every SSH login (Ansible, CI deploys) gets its own session-N.scope:
+  // one stream each blew Loki's active stream limit.
+  rule {
+    source_labels = ["unit"]
+    regex         = "session-\\d+\\.scope"
+    target_label  = "unit"
+    replacement   = "session.scope"
+  }
   rule {
     source_labels = ["__journal_priority_keyword"]
     target_label  = "level"
diff --git a/ansible/roles/monitoring_agent/templates/docker-compose.yml.j2 b/ansible/roles/monitoring_agent/templates/docker-compose.yml.j2
index 60e43be..eb0524b 100644
--- a/ansible/roles/monitoring_agent/templates/docker-compose.yml.j2
+++ b/ansible/roles/monitoring_agent/templates/docker-compose.yml.j2
@@ -37,6 +37,12 @@ services:
       - /sys:/sys:ro
       - /var/lib/docker/:/var/lib/docker:ro
       - /dev/disk/:/dev/disk:ro
+    healthcheck:
+      # The image probes localhost:8080; we listen on the VPC address only.
+      test: ["CMD", "wget", "-q", "--tries=1", "--spider", "http://{{ vpc_ip }}:{{ monitoring_cadvisor_port }}/healthz"]
+      interval: 30s
+      timeout: 3s
+      retries: 3
     restart: unless-stopped
     mem_limit: 192m