All checks were successful
deploy / ansible (push) Successful in 7m23s
Nothing watched the servers: tonight's asxpio crash loop was found by hand. On asxpio-ops, Prometheus (30 days), Loki (14 days) and Grafana at gf.asxp.io (WireGuard only), with blackbox_exporter probing every service from outside and postgres_exporter. On both hosts node_exporter and cAdvisor on the VPC address, and Alloy shipping container logs, the journal and Traefik's access log to Loki over the VPC. Traefik and Forgejo expose their metrics; restic and the bucket replica record their last success for node_exporter. Dashboards: an overview as home, plus hosts, containers, Traefik, probes, Postgres and logs. Alerts by email to me@asxp.io: host or service down, certificate under 14 days, disk or memory nearly full, a container restart-looping, Postgres down, backup or replica stale. Assisted-by: Claude:opus-5-5
1504 lines
No EOL
26 KiB
JSON
1504 lines
No EOL
26 KiB
JSON
{
|
|
"uid": "asxpio-overview",
|
|
"title": "asxpio overview",
|
|
"tags": [
|
|
"asxpio"
|
|
],
|
|
"timezone": "browser",
|
|
"editable": false,
|
|
"refresh": "1m",
|
|
"time": {
|
|
"from": "now-6h",
|
|
"to": "now"
|
|
},
|
|
"schemaVersion": 39,
|
|
"panels": [
|
|
{
|
|
"type": "row",
|
|
"title": "Status",
|
|
"id": 1,
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"panels": []
|
|
},
|
|
{
|
|
"type": "stat",
|
|
"title": "Services (outside-in)",
|
|
"id": 2,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "probe_success",
|
|
"legendFormat": "{{probe}}",
|
|
"refId": "A",
|
|
"instant": true,
|
|
"range": false
|
|
}
|
|
],
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 16,
|
|
"x": 0,
|
|
"y": 1
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "red",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "green",
|
|
"value": 1
|
|
}
|
|
]
|
|
},
|
|
"mappings": [
|
|
{
|
|
"type": "value",
|
|
"options": {
|
|
"0": {
|
|
"text": "DOWN",
|
|
"color": "red"
|
|
},
|
|
"1": {
|
|
"text": "UP",
|
|
"color": "green"
|
|
}
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "background",
|
|
"graphMode": "none",
|
|
"textMode": "value_and_name",
|
|
"justifyMode": "center",
|
|
"orientation": "auto",
|
|
"text": {
|
|
"titleSize": 12,
|
|
"valueSize": 16
|
|
}
|
|
},
|
|
"description": "blackbox_exporter probes from asxpio-ops: HTTPS sites, XMPP, TURN, Xray, Minecraft, SillyTavern"
|
|
},
|
|
{
|
|
"type": "bargauge",
|
|
"title": "Certificates: days left",
|
|
"id": 3,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "(probe_ssl_earliest_cert_expiry - time()) / 86400",
|
|
"legendFormat": "{{probe}}",
|
|
"refId": "A",
|
|
"instant": true,
|
|
"range": false
|
|
}
|
|
],
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 1
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"unit": "d",
|
|
"decimals": 0,
|
|
"min": 0,
|
|
"max": 90,
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "red",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "orange",
|
|
"value": 14
|
|
},
|
|
{
|
|
"color": "green",
|
|
"value": 30
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"displayMode": "basic",
|
|
"orientation": "horizontal",
|
|
"showUnfilled": true,
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"values": false
|
|
},
|
|
"minVizHeight": 10
|
|
}
|
|
},
|
|
{
|
|
"type": "row",
|
|
"title": "Hosts",
|
|
"id": 4,
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 8
|
|
},
|
|
"panels": []
|
|
},
|
|
{
|
|
"type": "gauge",
|
|
"title": "CPU",
|
|
"id": 5,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "100 - avg by (host) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100",
|
|
"legendFormat": "{{host}}",
|
|
"refId": "A",
|
|
"instant": true,
|
|
"range": false
|
|
}
|
|
],
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 4,
|
|
"x": 0,
|
|
"y": 9
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"unit": "percent",
|
|
"decimals": 0,
|
|
"min": 0,
|
|
"max": 100,
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "orange",
|
|
"value": 75
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 90
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"values": false
|
|
},
|
|
"showThresholdMarkers": true
|
|
}
|
|
},
|
|
{
|
|
"type": "gauge",
|
|
"title": "Memory",
|
|
"id": 6,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)",
|
|
"legendFormat": "{{host}}",
|
|
"refId": "A",
|
|
"instant": true,
|
|
"range": false
|
|
}
|
|
],
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 4,
|
|
"x": 4,
|
|
"y": 9
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"unit": "percent",
|
|
"decimals": 0,
|
|
"min": 0,
|
|
"max": 100,
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "orange",
|
|
"value": 75
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 90
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"values": false
|
|
},
|
|
"showThresholdMarkers": true
|
|
}
|
|
},
|
|
{
|
|
"type": "gauge",
|
|
"title": "Disk /",
|
|
"id": 7,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "100 * (1 - node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"})",
|
|
"legendFormat": "{{host}}",
|
|
"refId": "A",
|
|
"instant": true,
|
|
"range": false
|
|
}
|
|
],
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 4,
|
|
"x": 8,
|
|
"y": 9
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"unit": "percent",
|
|
"decimals": 0,
|
|
"min": 0,
|
|
"max": 100,
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "orange",
|
|
"value": 75
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 90
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"values": false
|
|
},
|
|
"showThresholdMarkers": true
|
|
}
|
|
},
|
|
{
|
|
"type": "gauge",
|
|
"title": "Load (1m) per CPU",
|
|
"id": 8,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "node_load1 / count by (host) (node_cpu_seconds_total{mode=\"idle\"})",
|
|
"legendFormat": "{{host}}",
|
|
"refId": "A",
|
|
"instant": true,
|
|
"range": false
|
|
}
|
|
],
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 4,
|
|
"x": 12,
|
|
"y": 9
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"unit": "percentunit",
|
|
"decimals": 0,
|
|
"min": 0,
|
|
"max": 2,
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "orange",
|
|
"value": 0.8
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 1.2
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"values": false
|
|
},
|
|
"showThresholdMarkers": true
|
|
}
|
|
},
|
|
{
|
|
"type": "stat",
|
|
"title": "Uptime",
|
|
"id": 9,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "time() - node_boot_time_seconds",
|
|
"legendFormat": "{{host}}",
|
|
"refId": "A",
|
|
"instant": true,
|
|
"range": false
|
|
}
|
|
],
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 4,
|
|
"x": 16,
|
|
"y": 9
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"unit": "s",
|
|
"decimals": 0,
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "blue",
|
|
"value": null
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "none",
|
|
"textMode": "value_and_name",
|
|
"justifyMode": "center",
|
|
"orientation": "auto"
|
|
}
|
|
},
|
|
{
|
|
"type": "stat",
|
|
"title": "Swap used",
|
|
"id": 10,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "node_memory_SwapTotal_bytes - node_memory_SwapFree_bytes",
|
|
"legendFormat": "{{host}}",
|
|
"refId": "A",
|
|
"instant": true,
|
|
"range": false
|
|
}
|
|
],
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 4,
|
|
"x": 20,
|
|
"y": 9
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"unit": "bytes",
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "orange",
|
|
"value": 524288000
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 1572864000
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "none",
|
|
"textMode": "value_and_name",
|
|
"justifyMode": "center",
|
|
"orientation": "auto"
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "CPU by host",
|
|
"id": 11,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "100 - avg by (host) (rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])) * 100",
|
|
"legendFormat": "{{host}}",
|
|
"refId": "A",
|
|
"instant": false,
|
|
"range": true
|
|
}
|
|
],
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 15
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"unit": "percent",
|
|
"min": 0,
|
|
"max": 100
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "Memory used by host",
|
|
"id": 12,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes",
|
|
"legendFormat": "{{host}}",
|
|
"refId": "A",
|
|
"instant": false,
|
|
"range": true
|
|
}
|
|
],
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 15
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"unit": "bytes"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "Network by host (in +, out -)",
|
|
"id": 13,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum by (host) (rate(node_network_receive_bytes_total{device!~\"lo|veth.*|br-.*|docker.*\"}[$__rate_interval]))",
|
|
"legendFormat": "{{host}} in",
|
|
"refId": "A",
|
|
"instant": false,
|
|
"range": true
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "-sum by (host) (rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|br-.*|docker.*\"}[$__rate_interval]))",
|
|
"legendFormat": "{{host}} out",
|
|
"refId": "B",
|
|
"instant": false,
|
|
"range": true
|
|
}
|
|
],
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 15
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"unit": "Bps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "row",
|
|
"title": "Containers",
|
|
"id": 14,
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 23
|
|
},
|
|
"panels": []
|
|
},
|
|
{
|
|
"type": "bargauge",
|
|
"title": "Memory (top 12)",
|
|
"id": 15,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "topk(12, container_memory_working_set_bytes{name!=\"\"})",
|
|
"legendFormat": "{{host}} \u00b7 {{name}}",
|
|
"refId": "A",
|
|
"instant": true,
|
|
"range": false
|
|
}
|
|
],
|
|
"gridPos": {
|
|
"h": 10,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 24
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"unit": "bytes",
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "blue",
|
|
"value": null
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"displayMode": "gradient",
|
|
"orientation": "horizontal",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"values": false
|
|
},
|
|
"minVizHeight": 10
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "CPU (top 8)",
|
|
"id": 16,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "topk(8, sum by (host, name) (rate(container_cpu_usage_seconds_total{name!=\"\"}[$__rate_interval])))",
|
|
"legendFormat": "{{host}} \u00b7 {{name}}",
|
|
"refId": "A",
|
|
"instant": false,
|
|
"range": true
|
|
}
|
|
],
|
|
"gridPos": {
|
|
"h": 10,
|
|
"w": 10,
|
|
"x": 8,
|
|
"y": 24
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"unit": "percentunit"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "right",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
],
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "table",
|
|
"title": "Restarts (24 h)",
|
|
"id": 17,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sort_desc(sum by (host, name) (changes(container_start_time_seconds{name!=\"\"}[24h])) > 0)",
|
|
"legendFormat": "",
|
|
"refId": "A",
|
|
"instant": true,
|
|
"range": false
|
|
}
|
|
],
|
|
"gridPos": {
|
|
"h": 10,
|
|
"w": 6,
|
|
"x": 18,
|
|
"y": 24
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "orange",
|
|
"value": 1
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 3
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "Value"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "displayName",
|
|
"value": "restarts"
|
|
},
|
|
{
|
|
"id": "custom.cellOptions",
|
|
"value": {
|
|
"type": "color-background"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "Time"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.hidden",
|
|
"value": true
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"options": {
|
|
"showHeader": true,
|
|
"cellHeight": "sm"
|
|
}
|
|
},
|
|
{
|
|
"type": "row",
|
|
"title": "Web traffic (Traefik)",
|
|
"id": 18,
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 34
|
|
},
|
|
"panels": []
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "Requests by router",
|
|
"id": 19,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum by (router) (rate(traefik_router_requests_total[$__rate_interval]))",
|
|
"legendFormat": "{{router}}",
|
|
"refId": "A",
|
|
"instant": false,
|
|
"range": true
|
|
}
|
|
],
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 35
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"unit": "reqps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "5xx by router",
|
|
"id": 20,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum by (router) (rate(traefik_router_requests_total{code=~\"5..\"}[$__rate_interval]))",
|
|
"legendFormat": "{{router}}",
|
|
"refId": "A",
|
|
"instant": false,
|
|
"range": true
|
|
}
|
|
],
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 35
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"unit": "reqps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "Latency p95 by router",
|
|
"id": 21,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "histogram_quantile(0.95, sum by (le, router) (rate(traefik_router_request_duration_seconds_bucket[$__rate_interval])))",
|
|
"legendFormat": "{{router}}",
|
|
"refId": "A",
|
|
"instant": false,
|
|
"range": true
|
|
}
|
|
],
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 35
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "row",
|
|
"title": "Data and backups",
|
|
"id": 22,
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 43
|
|
},
|
|
"panels": []
|
|
},
|
|
{
|
|
"type": "stat",
|
|
"title": "Postgres",
|
|
"id": 23,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "pg_up",
|
|
"legendFormat": "up",
|
|
"refId": "A",
|
|
"instant": true,
|
|
"range": false
|
|
}
|
|
],
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 4,
|
|
"x": 0,
|
|
"y": 44
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "red",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "green",
|
|
"value": 1
|
|
}
|
|
]
|
|
},
|
|
"mappings": [
|
|
{
|
|
"type": "value",
|
|
"options": {
|
|
"0": {
|
|
"text": "DOWN",
|
|
"color": "red"
|
|
},
|
|
"1": {
|
|
"text": "UP",
|
|
"color": "green"
|
|
}
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "background",
|
|
"graphMode": "none",
|
|
"textMode": "value_and_name",
|
|
"justifyMode": "center",
|
|
"orientation": "auto"
|
|
}
|
|
},
|
|
{
|
|
"type": "stat",
|
|
"title": "asxpio DB size",
|
|
"id": 24,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "pg_database_size_bytes{datname=\"asxpio\"}",
|
|
"legendFormat": "asxpio",
|
|
"refId": "A",
|
|
"instant": true,
|
|
"range": false
|
|
}
|
|
],
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 4,
|
|
"x": 4,
|
|
"y": 44
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"unit": "bytes",
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "blue",
|
|
"value": null
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "none",
|
|
"textMode": "value_and_name",
|
|
"justifyMode": "center",
|
|
"orientation": "auto"
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "Postgres connections",
|
|
"id": 25,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum by (datname) (pg_stat_activity_count)",
|
|
"legendFormat": "{{datname}}",
|
|
"refId": "A",
|
|
"instant": false,
|
|
"range": true
|
|
}
|
|
],
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 6,
|
|
"x": 8,
|
|
"y": 44
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "bargauge",
|
|
"title": "Last good backup / replica (hours ago)",
|
|
"id": 26,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "(time() - restic_backup_last_success_timestamp_seconds) / 3600",
|
|
"legendFormat": "restic \u00b7 {{host}} \u00b7 {{repo}}",
|
|
"refId": "A",
|
|
"instant": true,
|
|
"range": false
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "(time() - s3_replicate_last_success_timestamp_seconds) / 3600",
|
|
"legendFormat": "replica hel1 \u2192 fsn1",
|
|
"refId": "B",
|
|
"instant": true,
|
|
"range": false
|
|
}
|
|
],
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 10,
|
|
"x": 14,
|
|
"y": 44
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"unit": "h",
|
|
"decimals": 1,
|
|
"min": 0,
|
|
"max": 48,
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "orange",
|
|
"value": 26
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 48
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"displayMode": "basic",
|
|
"orientation": "horizontal",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"values": false
|
|
},
|
|
"showUnfilled": true
|
|
}
|
|
},
|
|
{
|
|
"type": "row",
|
|
"title": "Logs",
|
|
"id": 27,
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 50
|
|
},
|
|
"panels": []
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "Log lines by host",
|
|
"id": 28,
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "sum by (host) (count_over_time({job=~\"docker|journal\"}[$__interval]))",
|
|
"legendFormat": "{{host}}",
|
|
"refId": "A",
|
|
"instant": false,
|
|
"range": true
|
|
}
|
|
],
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 51
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
},
|
|
"drawStyle": "bars"
|
|
}
|
|
},
|
|
{
|
|
"type": "logs",
|
|
"title": "Errors (all containers)",
|
|
"id": 29,
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "{job=\"docker\"} |~ \"(?i)(error|fatal|panic|exception|traceback)\" != \"level=info\"",
|
|
"legendFormat": "",
|
|
"refId": "A",
|
|
"instant": false,
|
|
"range": true
|
|
}
|
|
],
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 16,
|
|
"x": 8,
|
|
"y": 51
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"showTime": true,
|
|
"wrapLogMessage": true,
|
|
"sortOrder": "Descending",
|
|
"enableLogDetails": true,
|
|
"dedupStrategy": "signature"
|
|
}
|
|
}
|
|
],
|
|
"graphTooltip": 1,
|
|
"links": [
|
|
{
|
|
"title": "Dashboards",
|
|
"type": "dashboards",
|
|
"asDropdown": true,
|
|
"includeVars": false,
|
|
"keepTime": true,
|
|
"tags": []
|
|
}
|
|
]
|
|
} |