From 27d4b73eb00c14e8ff0293c84a28475ebbd0ce84 Mon Sep 17 00:00:00 2001 From: Sergei Poljanski Date: Fri, 13 Nov 2026 07:04:00 +0400 Subject: [PATCH] monitoring: keep Loki chunks in S3, 90 days of logs and metrics ops has 12 GB free on a 38 GB disk and the Traefik access log alone is ~600 MB, so logs can't grow there. From 2026-11-14 Loki writes chunks and index to Hetzner Object Storage (asxpio-loki-main, hel1) with the existing project key, passed through loki.env and -config.expand-env. Today's filesystem period ages out by retention. Log retention goes from 14 to 90 days. Prometheus metrics are small (~20 MB so far) and stay local, now for 90 days instead of 30. Assisted-by: Claude:opus-5-5 --- .../inventory/group_vars/all/monitoring.yaml | 4 ++-- ansible/inventory/group_vars/all/s3.yaml | 1 + ansible/roles/monitoring/tasks/main.yaml | 1 + .../monitoring/templates/docker-compose.yml.j2 | 4 +++- ansible/roles/monitoring/templates/loki.env.j2 | 3 +++ ansible/roles/monitoring/templates/loki.yml.j2 | 18 +++++++++++++++++- 6 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 ansible/roles/monitoring/templates/loki.env.j2 diff --git a/ansible/inventory/group_vars/all/monitoring.yaml b/ansible/inventory/group_vars/all/monitoring.yaml index 92ed2a7..a1e584d 100644 --- a/ansible/inventory/group_vars/all/monitoring.yaml +++ b/ansible/inventory/group_vars/all/monitoring.yaml @@ -19,8 +19,8 @@ monitoring_blackbox_version: "v0.29.1" # renovate: datasource=docker depName=quay.io/prometheuscommunity/postgres-exporter monitoring_postgres_exporter_version: "v0.20.1" -monitoring_metrics_retention: 30d -monitoring_logs_retention: 336h # 14 days +monitoring_metrics_retention: 90d +monitoring_logs_retention: 2160h # 90 days, chunks in S3 (asxpio-loki-main) monitoring_alert_email: me@asxp.io # Agents listen on the host's VPC address only diff --git a/ansible/inventory/group_vars/all/s3.yaml b/ansible/inventory/group_vars/all/s3.yaml index 954951d..2ebea3b 100644 --- a/ansible/inventory/group_vars/all/s3.yaml +++ b/ansible/inventory/group_vars/all/s3.yaml @@ -17,6 +17,7 @@ s3_bucket_terraform_main: asxpio-terraform-main s3_bucket_terraform_repl: asxpio-terraform-repl s3_bucket_backups_main: asxpio-backups-main s3_bucket_backups_repl: asxpio-backups-repl +s3_bucket_loki: asxpio-loki-main # S3 Buckets to mount on servers (non-replication only) s3_buckets: diff --git a/ansible/roles/monitoring/tasks/main.yaml b/ansible/roles/monitoring/tasks/main.yaml index 5454b40..5e8f643 100644 --- a/ansible/roles/monitoring/tasks/main.yaml +++ b/ansible/roles/monitoring/tasks/main.yaml @@ -30,6 +30,7 @@ - { src: blackbox.yml.j2, dest: blackbox.yml } - { src: loki.yml.j2, dest: loki.yml } - { src: grafana.env.j2, dest: grafana.env, mode: "0600" } + - { src: loki.env.j2, dest: loki.env, mode: "0600" } - { src: datasources.yaml.j2, dest: grafana/provisioning/datasources/datasources.yaml } - { src: dashboards.yaml.j2, dest: grafana/provisioning/dashboards/dashboards.yaml } - { src: alerting.yaml.j2, dest: grafana/provisioning/alerting/alerting.yaml } diff --git a/ansible/roles/monitoring/templates/docker-compose.yml.j2 b/ansible/roles/monitoring/templates/docker-compose.yml.j2 index 82a15c3..e6374ea 100644 --- a/ansible/roles/monitoring/templates/docker-compose.yml.j2 +++ b/ansible/roles/monitoring/templates/docker-compose.yml.j2 @@ -18,7 +18,9 @@ services: loki: image: grafana/loki:{{ monitoring_loki_version }} container_name: loki - command: ["-config.file=/etc/loki/loki.yml"] + # S3 keys come from loki.env, expanded into the config at start + command: ["-config.file=/etc/loki/loki.yml", "-config.expand-env=true"] + env_file: [loki.env] ports: # Alloy on both hosts pushes here over the VPC - "{{ asxpio_ops_vpc_ip }}:{{ monitoring_loki_port }}:3100" diff --git a/ansible/roles/monitoring/templates/loki.env.j2 b/ansible/roles/monitoring/templates/loki.env.j2 new file mode 100644 index 0000000..ca8e610 --- /dev/null +++ b/ansible/roles/monitoring/templates/loki.env.j2 @@ -0,0 +1,3 @@ +# Managed by Ansible (role monitoring). +S3_ACCESS_KEY={{ s3_access_key }} +S3_SECRET_KEY={{ s3_secret_key }} diff --git a/ansible/roles/monitoring/templates/loki.yml.j2 b/ansible/roles/monitoring/templates/loki.yml.j2 index 15f67fe..e47ee45 100644 --- a/ansible/roles/monitoring/templates/loki.yml.j2 +++ b/ansible/roles/monitoring/templates/loki.yml.j2 @@ -22,6 +22,22 @@ schema_config: index: prefix: index_ period: 24h + # From here on chunks and index live in Hetzner Object Storage, not on + # ops' small disk. The filesystem period above ages out with retention. + - from: "2026-11-14" + store: tsdb + object_store: aws + schema: v13 + index: + prefix: index_ + period: 24h +storage_config: + aws: + endpoint: {{ s3_region }}.your-objectstorage.com + region: {{ s3_region }} + bucketnames: {{ s3_bucket_loki }} + access_key_id: ${S3_ACCESS_KEY} + secret_access_key: ${S3_SECRET_KEY} limits_config: retention_period: {{ monitoring_logs_retention }} reject_old_samples: true @@ -32,6 +48,6 @@ limits_config: compactor: working_directory: /loki/compactor retention_enabled: true - delete_request_store: filesystem + delete_request_store: aws analytics: reporting_enabled: false