monitoring: keep Loki chunks in S3, 90 days of logs and metrics

ops has 12 GB free on a 38 GB disk and the Traefik access log alone
is ~600 MB, so logs can't grow there. From 2026-11-14 Loki writes
chunks and index to Hetzner Object Storage (asxpio-loki-main, hel1)
with the existing project key, passed through loki.env and
-config.expand-env. Today's filesystem period ages out by retention.

Log retention goes from 14 to 90 days. Prometheus metrics are small
(~20 MB so far) and stay local, now for 90 days instead of 30.

Assisted-by: Claude:opus-5-5

ansible/inventory/group_vars/all/monitoring.yaml

@@ -19,8 +19,8 @@ monitoring_blackbox_version: "v0.29.1"
 # renovate: datasource=docker depName=quay.io/prometheuscommunity/postgres-exporter
 monitoring_postgres_exporter_version: "v0.20.1"
 
-monitoring_metrics_retention: 30d
-monitoring_logs_retention: 336h   # 14 days
+monitoring_metrics_retention: 90d
+monitoring_logs_retention: 2160h  # 90 days, chunks in S3 (asxpio-loki-main)
 monitoring_alert_email: me@asxp.io
 
 # Agents listen on the host's VPC address only

ansible/inventory/group_vars/all/s3.yaml

@@ -17,6 +17,7 @@ s3_bucket_terraform_main: asxpio-terraform-main
 s3_bucket_terraform_repl: asxpio-terraform-repl
 s3_bucket_backups_main: asxpio-backups-main
 s3_bucket_backups_repl: asxpio-backups-repl
+s3_bucket_loki: asxpio-loki-main
 
 # S3 Buckets to mount on servers (non-replication only)
 s3_buckets:

ansible/roles/monitoring/tasks/main.yaml

@@ -30,6 +30,7 @@
       - { src: blackbox.yml.j2, dest: blackbox.yml }
       - { src: loki.yml.j2, dest: loki.yml }
       - { src: grafana.env.j2, dest: grafana.env, mode: "0600" }
+      - { src: loki.env.j2, dest: loki.env, mode: "0600" }
       - { src: datasources.yaml.j2, dest: grafana/provisioning/datasources/datasources.yaml }
       - { src: dashboards.yaml.j2, dest: grafana/provisioning/dashboards/dashboards.yaml }
       - { src: alerting.yaml.j2, dest: grafana/provisioning/alerting/alerting.yaml }

ansible/roles/monitoring/templates/docker-compose.yml.j2

@@ -18,7 +18,9 @@ services:
   loki:
     image: grafana/loki:{{ monitoring_loki_version }}
     container_name: loki
-    command: ["-config.file=/etc/loki/loki.yml"]
+    # S3 keys come from loki.env, expanded into the config at start
+    command: ["-config.file=/etc/loki/loki.yml", "-config.expand-env=true"]
+    env_file: [loki.env]
     ports:
       # Alloy on both hosts pushes here over the VPC
       - "{{ asxpio_ops_vpc_ip }}:{{ monitoring_loki_port }}:3100"

ansible/roles/monitoring/templates/loki.env.j2

@@ -0,0 +1,3 @@
+# Managed by Ansible (role monitoring).
+S3_ACCESS_KEY={{ s3_access_key }}
+S3_SECRET_KEY={{ s3_secret_key }}

ansible/roles/monitoring/templates/loki.yml.j2

@@ -22,6 +22,22 @@ schema_config:
       index:
         prefix: index_
         period: 24h
+    # From here on chunks and index live in Hetzner Object Storage, not on
+    # ops' small disk. The filesystem period above ages out with retention.
+    - from: "2026-11-14"
+      store: tsdb
+      object_store: aws
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+storage_config:
+  aws:
+    endpoint: {{ s3_region }}.your-objectstorage.com
+    region: {{ s3_region }}
+    bucketnames: {{ s3_bucket_loki }}
+    access_key_id: ${S3_ACCESS_KEY}
+    secret_access_key: ${S3_SECRET_KEY}
 limits_config:
   retention_period: {{ monitoring_logs_retention }}
   reject_old_samples: true
@@ -32,6 +48,6 @@ limits_config:
 compactor:
   working_directory: /loki/compactor
   retention_enabled: true
-  delete_request_store: filesystem
+  delete_request_store: aws
 analytics:
   reporting_enabled: false