ansible: monitoring with Prometheus, Loki and Grafana

Nothing watched the servers: tonight's asxpio crash loop was found by
hand. On asxpio-ops, Prometheus (30 days), Loki (14 days) and Grafana
at gf.asxp.io (WireGuard only), with blackbox_exporter probing every
service from outside and postgres_exporter. On both hosts node_exporter
and cAdvisor on the VPC address, and Alloy shipping container logs, the
journal and Traefik's access log to Loki over the VPC.

Traefik and Forgejo expose their metrics; restic and the bucket replica
record their last success for node_exporter. Dashboards: an overview as
home, plus hosts, containers, Traefik, probes, Postgres and logs.
Alerts by email to me@asxp.io: host or service down, certificate under
14 days, disk or memory nearly full, a container restart-looping,
Postgres down, backup or replica stale.

Assisted-by: Claude:opus-5-5

ansible/roles/monitoring/files/dashboards/Logs/logs.json

@@ -0,0 +1,248 @@
+{
+ "annotations": {
+  "list": [
+   {
+    "$$hashKey": "object:75",
+    "builtIn": 1,
+    "datasource": "-- Grafana --",
+    "enable": true,
+    "hide": true,
+    "iconColor": "rgba(0, 211, 255, 1)",
+    "name": "Annotations & Alerts",
+    "type": "dashboard"
+   }
+  ]
+ },
+ "description": "Log Viewer Dashboard for Loki",
+ "editable": false,
+ "gnetId": 13639,
+ "graphTooltip": 0,
+ "id": null,
+ "iteration": 1608932746420,
+ "links": [
+  {
+   "$$hashKey": "object:59",
+   "icon": "bolt",
+   "includeVars": true,
+   "keepTime": true,
+   "tags": [],
+   "targetBlank": true,
+   "title": "View In Explore",
+   "type": "link",
+   "url": "/explore?orgId=1&left=[\"now-1h\",\"now\",\"Loki\",{\"expr\":\"{job=\\\"$app\\\"}\"},{\"ui\":[true,true,true,\"none\"]}]"
+  },
+  {
+   "$$hashKey": "object:61",
+   "icon": "external link",
+   "tags": [],
+   "targetBlank": true,
+   "title": "Learn LogQL",
+   "type": "link",
+   "url": "https://grafana.com/docs/loki/latest/logql/"
+  }
+ ],
+ "panels": [
+  {
+   "aliasColors": {},
+   "bars": true,
+   "dashLength": 10,
+   "dashes": false,
+   "datasource": "loki",
+   "fieldConfig": {
+    "defaults": {
+     "custom": {},
+     "links": []
+    },
+    "overrides": []
+   },
+   "fill": 1,
+   "fillGradient": 0,
+   "gridPos": {
+    "h": 3,
+    "w": 24,
+    "x": 0,
+    "y": 0
+   },
+   "hiddenSeries": false,
+   "id": 6,
+   "legend": {
+    "avg": false,
+    "current": false,
+    "max": false,
+    "min": false,
+    "show": false,
+    "total": false,
+    "values": false
+   },
+   "lines": false,
+   "linewidth": 1,
+   "nullPointMode": "null",
+   "percentage": false,
+   "pluginVersion": "7.1.0",
+   "pointradius": 2,
+   "points": false,
+   "renderer": "flot",
+   "seriesOverrides": [],
+   "spaceLength": 10,
+   "stack": false,
+   "steppedLine": false,
+   "targets": [
+    {
+     "expr": "sum(count_over_time({job=\"$app\"} |= \"$search\" [$__interval]))",
+     "legendFormat": "",
+     "refId": "A"
+    }
+   ],
+   "thresholds": [],
+   "timeFrom": null,
+   "timeRegions": [],
+   "timeShift": null,
+   "title": "",
+   "tooltip": {
+    "shared": true,
+    "sort": 0,
+    "value_type": "individual"
+   },
+   "type": "graph",
+   "xaxis": {
+    "buckets": null,
+    "mode": "time",
+    "name": null,
+    "show": true,
+    "values": []
+   },
+   "yaxes": [
+    {
+     "$$hashKey": "object:168",
+     "format": "short",
+     "label": null,
+     "logBase": 1,
+     "max": null,
+     "min": null,
+     "show": false
+    },
+    {
+     "$$hashKey": "object:169",
+     "format": "short",
+     "label": null,
+     "logBase": 1,
+     "max": null,
+     "min": null,
+     "show": false
+    }
+   ],
+   "yaxis": {
+    "align": false,
+    "alignLevel": null
+   }
+  },
+  {
+   "datasource": "loki",
+   "fieldConfig": {
+    "defaults": {
+     "custom": {}
+    },
+    "overrides": []
+   },
+   "gridPos": {
+    "h": 25,
+    "w": 24,
+    "x": 0,
+    "y": 3
+   },
+   "id": 2,
+   "maxDataPoints": "",
+   "options": {
+    "showLabels": false,
+    "showTime": true,
+    "sortOrder": "Descending",
+    "wrapLogMessage": false
+   },
+   "targets": [
+    {
+     "expr": "{job=\"$app\"} |= \"$search\" | logfmt",
+     "hide": false,
+     "legendFormat": "",
+     "refId": "A"
+    }
+   ],
+   "timeFrom": null,
+   "timeShift": null,
+   "title": "",
+   "transparent": true,
+   "type": "logs"
+  }
+ ],
+ "refresh": false,
+ "schemaVersion": 26,
+ "style": "dark",
+ "tags": [],
+ "templating": {
+  "list": [
+   {
+    "allValue": null,
+    "current": {},
+    "datasource": "loki",
+    "definition": "label_values(job)",
+    "hide": 0,
+    "includeAll": false,
+    "label": "App",
+    "multi": false,
+    "name": "app",
+    "options": [],
+    "query": "label_values(job)",
+    "refresh": 1,
+    "regex": "",
+    "skipUrlSync": false,
+    "sort": 0,
+    "tagValuesQuery": "",
+    "tags": [],
+    "tagsQuery": "",
+    "type": "query",
+    "useTags": false
+   },
+   {
+    "current": {
+     "selected": false,
+     "text": "",
+     "value": ""
+    },
+    "hide": 0,
+    "label": "String Match",
+    "name": "search",
+    "options": [
+     {
+      "selected": true,
+      "text": "",
+      "value": ""
+     }
+    ],
+    "query": "",
+    "skipUrlSync": false,
+    "type": "textbox"
+   }
+  ]
+ },
+ "time": {
+  "from": "now-1h",
+  "to": "now"
+ },
+ "timepicker": {
+  "hidden": false,
+  "refresh_intervals": [
+   "10s",
+   "30s",
+   "1m",
+   "5m",
+   "15m",
+   "30m",
+   "1h",
+   "2h",
+   "1d"
+  ]
+ },
+ "timezone": "",
+ "title": "Logs / App",
+ "uid": "gnet-13639",
+ "version": 13
+}