ansible: monitoring with Prometheus, Loki and Grafana
All checks were successful
deploy / ansible (push) Successful in 7m23s
All checks were successful
deploy / ansible (push) Successful in 7m23s
Nothing watched the servers: tonight's asxpio crash loop was found by hand. On asxpio-ops, Prometheus (30 days), Loki (14 days) and Grafana at gf.asxp.io (WireGuard only), with blackbox_exporter probing every service from outside and postgres_exporter. On both hosts node_exporter and cAdvisor on the VPC address, and Alloy shipping container logs, the journal and Traefik's access log to Loki over the VPC. Traefik and Forgejo expose their metrics; restic and the bucket replica record their last success for node_exporter. Dashboards: an overview as home, plus hosts, containers, Traefik, probes, Postgres and logs. Alerts by email to me@asxp.io: host or service down, certificate under 14 days, disk or memory nearly full, a container restart-looping, Postgres down, backup or replica stale. Assisted-by: Claude:opus-5-5
This commit is contained in:
parent
5289542dca
commit
e20f5a7d07
GPG key ID:
4F8851660FA4121B
28 changed files with 24705 additions and 161 deletions
|
|
@ -99,3 +99,8 @@ SHOW_FOOTER_VERSION = false
|
|||
SHOW_FOOTER_TEMPLATE_LOAD_TIME = false
|
||||
SHOW_FOOTER_POWERED_BY = false
|
||||
|
||||
|
||||
[metrics]
|
||||
; scraped by Prometheus on asxpio-ops over the traefik network
|
||||
ENABLED = true
|
||||
TOKEN = {{ monitoring_forgejo_metrics_token }}
|
||||
|
|
|
|||
24
ansible/roles/monitoring/defaults/main.yaml
Normal file
24
ansible/roles/monitoring/defaults/main.yaml
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
---
|
||||
monitoring_base_path: "{{ services_base_path }}/monitoring"
|
||||
|
||||
# Hosts with node_exporter, cAdvisor and Alloy, scraped over the VPC
|
||||
monitoring_hosts:
|
||||
- { name: asxpio-ops, address: "{{ asxpio_ops_vpc_ip }}" }
|
||||
- { name: asxpio-services, address: "{{ asxpio_services_vpc_ip }}" }
|
||||
|
||||
# Outside-in probes (blackbox_exporter). module: see blackbox.yml.j2
|
||||
monitoring_probes:
|
||||
- { name: asxp.io, target: "https://asxp.io/", module: http_2xx }
|
||||
- { name: blog, target: "https://blog.asxp.io/", module: http_2xx }
|
||||
- { name: czsk.it, target: "https://czsk.it/", module: http_2xx }
|
||||
- { name: datacenterbingo, target: "https://datacenterbingo.xyz/", module: http_2xx }
|
||||
- { name: datacenterbingo data.json, target: "https://datacenterbingo.xyz/data.json", module: http_2xx }
|
||||
- { name: forgejo, target: "https://git.asxp.io/", module: http_2xx }
|
||||
- { name: s3, target: "https://s3.asxp.io/", module: http_up }
|
||||
- { name: sillytavern, target: "http://{{ asxpio_services_wireguard_ip }}:8000/", module: http_up }
|
||||
- { name: xmpp c2s starttls, target: "xmpp.asxp.io:5222", module: xmpp_starttls }
|
||||
- { name: xmpp c2s tls, target: "xmpp.asxp.io:5223", module: tls_connect }
|
||||
- { name: xmpp s2s, target: "xmpp.asxp.io:5269", module: tcp_connect }
|
||||
- { name: turn, target: "xmpp.asxp.io:3478", module: tcp_connect }
|
||||
- { name: xray, target: "{{ hostvars['asxpio-services'].public_ip }}:443", module: tcp_connect }
|
||||
- { name: minecraft, target: "mc.asxp.io:25565", module: tcp_connect }
|
||||
|
|
@ -0,0 +1,781 @@
|
|||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": "-- Grafana --",
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"gnetId": 14282,
|
||||
"graphTooltip": 0,
|
||||
"id": null,
|
||||
"iteration": 1617715580880,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"collapsed": false,
|
||||
"datasource": "prometheus",
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 8,
|
||||
"panels": [],
|
||||
"title": "CPU",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 1
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 15,
|
||||
"legend": {
|
||||
"alignAsTable": true,
|
||||
"avg": true,
|
||||
"current": false,
|
||||
"max": true,
|
||||
"min": false,
|
||||
"rightSide": true,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": true
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"nullPointMode": "null as zero",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "7.4.5",
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": true,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(container_cpu_usage_seconds_total{instance=~\"$host\",name=~\"$container\",name=~\".+\"}[5m])) by (name) *100",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "CPU Usage",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"$$hashKey": "object:606",
|
||||
"format": "percent",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"$$hashKey": "object:607",
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"datasource": "prometheus",
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 8
|
||||
},
|
||||
"id": 11,
|
||||
"panels": [],
|
||||
"title": "Memory",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 9
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 9,
|
||||
"legend": {
|
||||
"alignAsTable": true,
|
||||
"avg": true,
|
||||
"current": false,
|
||||
"max": true,
|
||||
"min": false,
|
||||
"rightSide": true,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": true
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"nullPointMode": "null as zero",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "7.4.5",
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": true,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(container_memory_rss{instance=~\"$host\",name=~\"$container\",name=~\".+\"}) by (name)",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "Memory Usage",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"$$hashKey": "object:606",
|
||||
"format": "bytes",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"$$hashKey": "object:607",
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 9
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 14,
|
||||
"legend": {
|
||||
"alignAsTable": true,
|
||||
"avg": true,
|
||||
"current": false,
|
||||
"max": true,
|
||||
"min": false,
|
||||
"rightSide": true,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": true
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"nullPointMode": "null as zero",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "7.4.5",
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": true,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(container_memory_cache{instance=~\"$host\",name=~\"$container\",name=~\".+\"}) by (name)",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "Memory Cached",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"$$hashKey": "object:606",
|
||||
"format": "bytes",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"$$hashKey": "object:607",
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"datasource": "prometheus",
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 17
|
||||
},
|
||||
"id": 2,
|
||||
"panels": [],
|
||||
"title": "Network",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 18
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 4,
|
||||
"legend": {
|
||||
"alignAsTable": true,
|
||||
"avg": true,
|
||||
"current": false,
|
||||
"hideEmpty": false,
|
||||
"hideZero": false,
|
||||
"max": true,
|
||||
"min": false,
|
||||
"rightSide": true,
|
||||
"show": true,
|
||||
"sideWidth": null,
|
||||
"total": false,
|
||||
"values": true
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"nullPointMode": "null",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "7.4.5",
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(container_network_receive_bytes_total{instance=~\"$host\",name=~\"$container\",name=~\".+\"}[5m])) by (name)",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "Received Network Traffic",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"$$hashKey": "object:674",
|
||||
"format": "Bps",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"$$hashKey": "object:675",
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 18
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 6,
|
||||
"legend": {
|
||||
"alignAsTable": true,
|
||||
"avg": true,
|
||||
"current": false,
|
||||
"max": true,
|
||||
"min": false,
|
||||
"rightSide": true,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": true
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"nullPointMode": "null",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "7.4.5",
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(container_network_transmit_bytes_total{instance=~\"$host\",name=~\"$container\",name=~\".+\"}[5m])) by (name)",
|
||||
"interval": "",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "Sent Network Traffic",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"$$hashKey": "object:832",
|
||||
"format": "Bps",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"$$hashKey": "object:833",
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"datasource": "prometheus",
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 26
|
||||
},
|
||||
"id": 19,
|
||||
"panels": [],
|
||||
"title": "Misc",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": "prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"align": null,
|
||||
"filterable": false
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "id"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "custom.width",
|
||||
"value": 260
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "Running"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "unit",
|
||||
"value": "d"
|
||||
},
|
||||
{
|
||||
"id": "decimals",
|
||||
"value": 1
|
||||
},
|
||||
{
|
||||
"id": "custom.displayMode",
|
||||
"value": "color-text"
|
||||
},
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"fixedColor": "dark-green",
|
||||
"mode": "fixed"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 10,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 27
|
||||
},
|
||||
"id": 17,
|
||||
"options": {
|
||||
"showHeader": true,
|
||||
"sortBy": []
|
||||
},
|
||||
"pluginVersion": "7.4.5",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(time() - container_start_time_seconds{instance=~\"$host\",name=~\"$container\",name=~\".+\"})/86400",
|
||||
"format": "table",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"timeFrom": null,
|
||||
"timeShift": null,
|
||||
"title": "Containers Info",
|
||||
"transformations": [
|
||||
{
|
||||
"id": "filterFieldsByName",
|
||||
"options": {
|
||||
"include": {
|
||||
"names": [
|
||||
"container_label_com_docker_compose_project",
|
||||
"container_label_com_docker_compose_project_working_dir",
|
||||
"image",
|
||||
"instance",
|
||||
"name",
|
||||
"Value",
|
||||
"container_label_com_docker_compose_service"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "organize",
|
||||
"options": {
|
||||
"excludeByName": {},
|
||||
"indexByName": {},
|
||||
"renameByName": {
|
||||
"Value": "Running",
|
||||
"container_label_com_docker_compose_project": "Label",
|
||||
"container_label_com_docker_compose_project_working_dir": "Working dir",
|
||||
"container_label_com_docker_compose_service": "Service",
|
||||
"image": "Registry Image",
|
||||
"instance": "Instance",
|
||||
"name": "Name"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"type": "table"
|
||||
}
|
||||
],
|
||||
"schemaVersion": 27,
|
||||
"style": "dark",
|
||||
"tags": [
|
||||
"cadvisor",
|
||||
"docker"
|
||||
],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"allValue": ".*",
|
||||
"current": {},
|
||||
"datasource": "prometheus",
|
||||
"definition": "label_values({__name__=~\"container.*\"},instance)",
|
||||
"description": null,
|
||||
"error": null,
|
||||
"hide": 0,
|
||||
"includeAll": true,
|
||||
"label": "Host",
|
||||
"multi": false,
|
||||
"name": "host",
|
||||
"options": [],
|
||||
"query": {
|
||||
"query": "label_values({__name__=~\"container.*\"},instance)",
|
||||
"refId": "Prometheus-host-Variable-Query"
|
||||
},
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
"sort": 5,
|
||||
"tagValuesQuery": "",
|
||||
"tags": [],
|
||||
"tagsQuery": "",
|
||||
"type": "query",
|
||||
"useTags": false
|
||||
},
|
||||
{
|
||||
"allValue": ".*",
|
||||
"current": {},
|
||||
"datasource": "prometheus",
|
||||
"definition": "label_values({__name__=~\"container.*\", instance=~\"$host\"},name)",
|
||||
"description": null,
|
||||
"error": null,
|
||||
"hide": 0,
|
||||
"includeAll": true,
|
||||
"label": "Container",
|
||||
"multi": false,
|
||||
"name": "container",
|
||||
"options": [],
|
||||
"query": {
|
||||
"query": "label_values({__name__=~\"container.*\", instance=~\"$host\"},name)",
|
||||
"refId": "Prometheus-container-Variable-Query"
|
||||
},
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
"sort": 0,
|
||||
"tagValuesQuery": "",
|
||||
"tags": [],
|
||||
"tagsQuery": "",
|
||||
"type": "query",
|
||||
"useTags": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {
|
||||
"from": "now-6h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "",
|
||||
"title": "Cadvisor exporter",
|
||||
"uid": "gnet-14282",
|
||||
"version": 1,
|
||||
"description": "Simple exporter for cadvisor only"
|
||||
}
|
||||
15517
ansible/roles/monitoring/files/dashboards/Infrastructure/hosts.json
Normal file
15517
ansible/roles/monitoring/files/dashboards/Infrastructure/hosts.json
Normal file
File diff suppressed because it is too large
Load diff
248
ansible/roles/monitoring/files/dashboards/Logs/logs.json
Normal file
248
ansible/roles/monitoring/files/dashboards/Logs/logs.json
Normal file
|
|
@ -0,0 +1,248 @@
|
|||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"$$hashKey": "object:75",
|
||||
"builtIn": 1,
|
||||
"datasource": "-- Grafana --",
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"description": "Log Viewer Dashboard for Loki",
|
||||
"editable": false,
|
||||
"gnetId": 13639,
|
||||
"graphTooltip": 0,
|
||||
"id": null,
|
||||
"iteration": 1608932746420,
|
||||
"links": [
|
||||
{
|
||||
"$$hashKey": "object:59",
|
||||
"icon": "bolt",
|
||||
"includeVars": true,
|
||||
"keepTime": true,
|
||||
"tags": [],
|
||||
"targetBlank": true,
|
||||
"title": "View In Explore",
|
||||
"type": "link",
|
||||
"url": "/explore?orgId=1&left=[\"now-1h\",\"now\",\"Loki\",{\"expr\":\"{job=\\\"$app\\\"}\"},{\"ui\":[true,true,true,\"none\"]}]"
|
||||
},
|
||||
{
|
||||
"$$hashKey": "object:61",
|
||||
"icon": "external link",
|
||||
"tags": [],
|
||||
"targetBlank": true,
|
||||
"title": "Learn LogQL",
|
||||
"type": "link",
|
||||
"url": "https://grafana.com/docs/loki/latest/logql/"
|
||||
}
|
||||
],
|
||||
"panels": [
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": true,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "loki",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {},
|
||||
"links": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"gridPos": {
|
||||
"h": 3,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 6,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": false,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": false,
|
||||
"linewidth": 1,
|
||||
"nullPointMode": "null",
|
||||
"percentage": false,
|
||||
"pluginVersion": "7.1.0",
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(count_over_time({job=\"$app\"} |= \"$search\" [$__interval]))",
|
||||
"legendFormat": "",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"$$hashKey": "object:168",
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": false
|
||||
},
|
||||
{
|
||||
"$$hashKey": "object:169",
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": false
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"datasource": "loki",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 25,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 3
|
||||
},
|
||||
"id": 2,
|
||||
"maxDataPoints": "",
|
||||
"options": {
|
||||
"showLabels": false,
|
||||
"showTime": true,
|
||||
"sortOrder": "Descending",
|
||||
"wrapLogMessage": false
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "{job=\"$app\"} |= \"$search\" | logfmt",
|
||||
"hide": false,
|
||||
"legendFormat": "",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"timeFrom": null,
|
||||
"timeShift": null,
|
||||
"title": "",
|
||||
"transparent": true,
|
||||
"type": "logs"
|
||||
}
|
||||
],
|
||||
"refresh": false,
|
||||
"schemaVersion": 26,
|
||||
"style": "dark",
|
||||
"tags": [],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"allValue": null,
|
||||
"current": {},
|
||||
"datasource": "loki",
|
||||
"definition": "label_values(job)",
|
||||
"hide": 0,
|
||||
"includeAll": false,
|
||||
"label": "App",
|
||||
"multi": false,
|
||||
"name": "app",
|
||||
"options": [],
|
||||
"query": "label_values(job)",
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
"sort": 0,
|
||||
"tagValuesQuery": "",
|
||||
"tags": [],
|
||||
"tagsQuery": "",
|
||||
"type": "query",
|
||||
"useTags": false
|
||||
},
|
||||
{
|
||||
"current": {
|
||||
"selected": false,
|
||||
"text": "",
|
||||
"value": ""
|
||||
},
|
||||
"hide": 0,
|
||||
"label": "String Match",
|
||||
"name": "search",
|
||||
"options": [
|
||||
{
|
||||
"selected": true,
|
||||
"text": "",
|
||||
"value": ""
|
||||
}
|
||||
],
|
||||
"query": "",
|
||||
"skipUrlSync": false,
|
||||
"type": "textbox"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {
|
||||
"hidden": false,
|
||||
"refresh_intervals": [
|
||||
"10s",
|
||||
"30s",
|
||||
"1m",
|
||||
"5m",
|
||||
"15m",
|
||||
"30m",
|
||||
"1h",
|
||||
"2h",
|
||||
"1d"
|
||||
]
|
||||
},
|
||||
"timezone": "",
|
||||
"title": "Logs / App",
|
||||
"uid": "gnet-13639",
|
||||
"version": 13
|
||||
}
|
||||
3131
ansible/roles/monitoring/files/dashboards/Services/postgres.json
Normal file
3131
ansible/roles/monitoring/files/dashboards/Services/postgres.json
Normal file
File diff suppressed because it is too large
Load diff
1071
ansible/roles/monitoring/files/dashboards/Services/probes.json
Normal file
1071
ansible/roles/monitoring/files/dashboards/Services/probes.json
Normal file
File diff suppressed because it is too large
Load diff
1527
ansible/roles/monitoring/files/dashboards/Services/traefik.json
Normal file
1527
ansible/roles/monitoring/files/dashboards/Services/traefik.json
Normal file
File diff suppressed because it is too large
Load diff
1504
ansible/roles/monitoring/files/dashboards/overview.json
Normal file
1504
ansible/roles/monitoring/files/dashboards/overview.json
Normal file
File diff suppressed because it is too large
Load diff
37
ansible/roles/monitoring/tasks/main.yaml
Normal file
37
ansible/roles/monitoring/tasks/main.yaml
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
---
|
||||
# Prometheus, Loki, Grafana, blackbox_exporter and postgres_exporter on
|
||||
# asxpio-ops. Grafana: https://gf.asxp.io, WireGuard only.
|
||||
- name: Create the provisioning directories
|
||||
ansible.builtin.file:
|
||||
path: "{{ monitoring_base_path }}/{{ item }}"
|
||||
state: directory
|
||||
mode: "0755"
|
||||
loop:
|
||||
- grafana/provisioning/datasources
|
||||
- grafana/provisioning/dashboards
|
||||
- grafana/provisioning/alerting
|
||||
- grafana/dashboards
|
||||
|
||||
- name: Deploy the dashboards
|
||||
ansible.builtin.copy:
|
||||
src: dashboards/
|
||||
dest: "{{ monitoring_base_path }}/grafana/dashboards/"
|
||||
mode: "0644"
|
||||
|
||||
- name: Deploy the monitoring stack
|
||||
ansible.builtin.include_role:
|
||||
name: docker_service
|
||||
vars:
|
||||
service_name: monitoring
|
||||
service_base_path: "{{ monitoring_base_path }}"
|
||||
service_templates:
|
||||
- { src: docker-compose.yml.j2, dest: docker-compose.yml }
|
||||
- { src: prometheus.yml.j2, dest: prometheus.yml }
|
||||
- { src: blackbox.yml.j2, dest: blackbox.yml }
|
||||
- { src: loki.yml.j2, dest: loki.yml }
|
||||
- { src: grafana.env.j2, dest: grafana.env, mode: "0600" }
|
||||
- { src: datasources.yaml.j2, dest: grafana/provisioning/datasources/datasources.yaml }
|
||||
- { src: dashboards.yaml.j2, dest: grafana/provisioning/dashboards/dashboards.yaml }
|
||||
- { src: alerting.yaml.j2, dest: grafana/provisioning/alerting/alerting.yaml }
|
||||
- { src: alert-rules.yaml.j2, dest: grafana/provisioning/alerting/rules.yaml }
|
||||
health_check_url: "http://{{ asxpio_ops_vpc_ip }}:{{ monitoring_loki_port }}/ready"
|
||||
170
ansible/roles/monitoring/templates/alert-rules.yaml.j2
Normal file
170
ansible/roles/monitoring/templates/alert-rules.yaml.j2
Normal file
|
|
@ -0,0 +1,170 @@
|
|||
# Managed by Ansible (role monitoring). Generated; see the commit adding it.
|
||||
{% raw %}
|
||||
apiVersion: 1
|
||||
groups:
|
||||
- orgId: 1
|
||||
name: asxpio
|
||||
folder: Alerts
|
||||
interval: 1m
|
||||
rules:
|
||||
- uid: host-down
|
||||
title: "Host down"
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 900, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model: { refId: A, expr: "1 - up{job=\"node\"}", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [0] } } ] }
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 2m
|
||||
labels: { severity: critical }
|
||||
annotations: { summary: "{{ $labels.host }} stopped answering (node_exporter over the VPC)" }
|
||||
- uid: target-down
|
||||
title: "Scrape target down"
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 900, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model: { refId: A, expr: "up{job!~\"node|probe_.*\"} == 0", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [-1] } } ] }
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 5m
|
||||
labels: { severity: warning }
|
||||
annotations: { summary: "{{ $labels.job }} on {{ $labels.instance }} is not answering metrics scrapes" }
|
||||
- uid: probe-down
|
||||
title: "Service down (outside-in probe)"
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 900, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model: { refId: A, expr: "1 - probe_success", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [0] } } ] }
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 2m
|
||||
labels: { severity: critical }
|
||||
annotations: { summary: "{{ $labels.probe }} ({{ $labels.instance }}) is not reachable" }
|
||||
- uid: cert-expiry
|
||||
title: "Certificate expires soon"
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 900, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model: { refId: A, expr: "(probe_ssl_earliest_cert_expiry - time()) / 86400", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: lt, params: [14] } } ] }
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 1h
|
||||
labels: { severity: warning }
|
||||
annotations: { summary: "{{ $labels.probe }}: certificate expires in {{ $values.A.Value | printf \"%.0f\" }} days" }
|
||||
- uid: disk-full
|
||||
title: "Disk almost full"
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 900, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model: { refId: A, expr: "100 * (1 - node_filesystem_avail_bytes{fstype!~\"tmpfs|overlay|squashfs|fuse.*\"} / node_filesystem_size_bytes{fstype!~\"tmpfs|overlay|squashfs|fuse.*\"})", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [85] } } ] }
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 10m
|
||||
labels: { severity: warning }
|
||||
annotations: { summary: "{{ $labels.host }} {{ $labels.mountpoint }} is {{ $values.A.Value | printf \"%.0f\" }}% full" }
|
||||
- uid: memory-low
|
||||
title: "Memory almost exhausted"
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 900, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model: { refId: A, expr: "100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [92] } } ] }
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 10m
|
||||
labels: { severity: warning }
|
||||
annotations: { summary: "{{ $labels.host }} memory is {{ $values.A.Value | printf \"%.0f\" }}% used" }
|
||||
- uid: container-restarting
|
||||
title: "Container restart-looping"
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 900, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model: { refId: A, expr: "changes(container_start_time_seconds{name!=\"\"}[15m])", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [2] } } ] }
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 0m
|
||||
labels: { severity: critical }
|
||||
annotations: { summary: "{{ $labels.name }} on {{ $labels.host }} restarted {{ $values.A.Value | printf \"%.0f\" }} times in 15 minutes" }
|
||||
- uid: postgres-down
|
||||
title: "Postgres down"
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 900, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model: { refId: A, expr: "pg_up", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: lt, params: [1] } } ] }
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 1m
|
||||
labels: { severity: critical }
|
||||
annotations: { summary: "Postgres (storage) is not answering" }
|
||||
- uid: backup-stale
|
||||
title: "Backup stale"
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 900, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model: { refId: A, expr: "(time() - restic_backup_last_success_timestamp_seconds) / 3600", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [26] } } ] }
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 0m
|
||||
labels: { severity: warning }
|
||||
annotations: { summary: "restic on {{ $labels.host }}: last good backup {{ $values.A.Value | printf \"%.0f\" }} hours ago" }
|
||||
- uid: replica-stale
|
||||
title: "Bucket replica stale"
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 900, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model: { refId: A, expr: "(time() - s3_replicate_last_success_timestamp_seconds) / 3600", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [26] } } ] }
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 0m
|
||||
labels: { severity: warning }
|
||||
annotations: { summary: "hel1 -> fsn1 bucket replica: last good run {{ $values.A.Value | printf \"%.0f\" }} hours ago" }
|
||||
{% endraw %}
|
||||
17
ansible/roles/monitoring/templates/alerting.yaml.j2
Normal file
17
ansible/roles/monitoring/templates/alerting.yaml.j2
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
apiVersion: 1
|
||||
contactPoints:
|
||||
- orgId: 1
|
||||
name: email
|
||||
receivers:
|
||||
- uid: email-me
|
||||
type: email
|
||||
settings:
|
||||
addresses: {{ monitoring_alert_email }}
|
||||
singleEmail: true
|
||||
policies:
|
||||
- orgId: 1
|
||||
receiver: email
|
||||
group_by: [alertname, host]
|
||||
group_wait: 1m
|
||||
group_interval: 5m
|
||||
repeat_interval: 4h
|
||||
38
ansible/roles/monitoring/templates/blackbox.yml.j2
Normal file
38
ansible/roles/monitoring/templates/blackbox.yml.j2
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
# Managed by Ansible (role monitoring).
|
||||
modules:
|
||||
http_2xx:
|
||||
prober: http
|
||||
timeout: 10s
|
||||
http:
|
||||
preferred_ip_protocol: ip4
|
||||
follow_redirects: true
|
||||
# Up means "answers": 401/403/404 are fine (s3 root, allowlisted UIs)
|
||||
http_up:
|
||||
prober: http
|
||||
timeout: 10s
|
||||
http:
|
||||
preferred_ip_protocol: ip4
|
||||
valid_status_codes: [200, 301, 302, 401, 403, 404]
|
||||
tls_connect:
|
||||
prober: tcp
|
||||
timeout: 10s
|
||||
tcp:
|
||||
preferred_ip_protocol: ip4
|
||||
tls: true
|
||||
tcp_connect:
|
||||
prober: tcp
|
||||
timeout: 10s
|
||||
tcp:
|
||||
preferred_ip_protocol: ip4
|
||||
# XMPP client port: stream header, STARTTLS, then the certificate
|
||||
xmpp_starttls:
|
||||
prober: tcp
|
||||
timeout: 10s
|
||||
tcp:
|
||||
preferred_ip_protocol: ip4
|
||||
query_response:
|
||||
- send: "<?xml version='1.0'?><stream:stream to='asxp.io' xmlns='jabber:client' xmlns:stream='http://etherx.jabber.org/streams' version='1.0'>"
|
||||
- expect: "urn:ietf:params:xml:ns:xmpp-tls"
|
||||
- send: "<starttls xmlns='urn:ietf:params:xml:ns:xmpp-tls'/>"
|
||||
- expect: "<proceed"
|
||||
- starttls: true
|
||||
9
ansible/roles/monitoring/templates/dashboards.yaml.j2
Normal file
9
ansible/roles/monitoring/templates/dashboards.yaml.j2
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
apiVersion: 1
|
||||
providers:
|
||||
- name: asxpio
|
||||
type: file
|
||||
disableDeletion: true
|
||||
allowUiUpdates: false
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
foldersFromFilesStructure: true
|
||||
14
ansible/roles/monitoring/templates/datasources.yaml.j2
Normal file
14
ansible/roles/monitoring/templates/datasources.yaml.j2
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
apiVersion: 1
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
jsonData: { timeInterval: 30s }
|
||||
- name: Loki
|
||||
uid: loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
87
ansible/roles/monitoring/templates/docker-compose.yml.j2
Normal file
87
ansible/roles/monitoring/templates/docker-compose.yml.j2
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
# Managed by Ansible (role monitoring).
|
||||
services:
|
||||
prometheus:
|
||||
image: prom/prometheus:{{ monitoring_prometheus_version }}
|
||||
container_name: prometheus
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time={{ monitoring_metrics_retention }}
|
||||
- --web.enable-lifecycle
|
||||
volumes:
|
||||
- ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- prometheus-data:/prometheus
|
||||
networks: [monitoring, traefik, storage]
|
||||
restart: unless-stopped
|
||||
mem_limit: 640m
|
||||
|
||||
loki:
|
||||
image: grafana/loki:{{ monitoring_loki_version }}
|
||||
container_name: loki
|
||||
command: ["-config.file=/etc/loki/loki.yml"]
|
||||
ports:
|
||||
# Alloy on both hosts pushes here over the VPC
|
||||
- "{{ asxpio_ops_vpc_ip }}:{{ monitoring_loki_port }}:3100"
|
||||
volumes:
|
||||
- ./loki.yml:/etc/loki/loki.yml:ro
|
||||
- loki-data:/loki
|
||||
networks: [monitoring]
|
||||
restart: unless-stopped
|
||||
mem_limit: 512m
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:{{ monitoring_grafana_version }}
|
||||
container_name: grafana
|
||||
env_file: [grafana.env]
|
||||
volumes:
|
||||
- grafana-data:/var/lib/grafana
|
||||
- ./grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
networks: [monitoring, traefik]
|
||||
restart: unless-stopped
|
||||
mem_limit: 384m
|
||||
labels:
|
||||
- traefik.enable=true
|
||||
- traefik.docker.network=traefik
|
||||
- traefik.http.routers.grafana.rule=Host(`{{ monitoring_grafana_host }}`)
|
||||
- traefik.http.routers.grafana.entrypoints=websecure
|
||||
- traefik.http.routers.grafana.tls=true
|
||||
- traefik.http.routers.grafana.tls.certresolver=letsencrypt
|
||||
- traefik.http.routers.grafana.middlewares=grafana-wireguard-only
|
||||
- traefik.http.middlewares.grafana-wireguard-only.ipallowlist.sourcerange={{ wireguard_network_cidr }}
|
||||
- traefik.http.services.grafana.loadbalancer.server.port=3000
|
||||
|
||||
blackbox:
|
||||
image: prom/blackbox-exporter:{{ monitoring_blackbox_version }}
|
||||
container_name: blackbox
|
||||
command: ["--config.file=/etc/blackbox/blackbox.yml"]
|
||||
volumes:
|
||||
- ./blackbox.yml:/etc/blackbox/blackbox.yml:ro
|
||||
networks: [monitoring]
|
||||
restart: unless-stopped
|
||||
mem_limit: 64m
|
||||
|
||||
postgres-exporter:
|
||||
image: quay.io/prometheuscommunity/postgres-exporter:{{ monitoring_postgres_exporter_version }}
|
||||
container_name: postgres-exporter
|
||||
# The password comes from the storage stack's own .env (same host)
|
||||
env_file: [{{ services_base_path }}/storage/.env]
|
||||
environment:
|
||||
DATA_SOURCE_URI: storage-postgres:5432/postgres?sslmode=disable
|
||||
DATA_SOURCE_USER: postgres
|
||||
entrypoint: ["/bin/sh", "-c", "DATA_SOURCE_PASS=\"$$POSTGRES_PASSWORD\" exec /bin/postgres_exporter"]
|
||||
networks: [monitoring, storage]
|
||||
restart: unless-stopped
|
||||
mem_limit: 64m
|
||||
|
||||
volumes:
|
||||
prometheus-data:
|
||||
loki-data:
|
||||
grafana-data:
|
||||
|
||||
networks:
|
||||
monitoring:
|
||||
traefik:
|
||||
external: true
|
||||
storage:
|
||||
external: true
|
||||
17
ansible/roles/monitoring/templates/grafana.env.j2
Normal file
17
ansible/roles/monitoring/templates/grafana.env.j2
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
GF_SERVER_ROOT_URL=https://{{ monitoring_grafana_host }}/
|
||||
GF_SECURITY_ADMIN_USER=admin
|
||||
GF_SECURITY_ADMIN_PASSWORD={{ monitoring_grafana_admin_password }}
|
||||
GF_SECURITY_COOKIE_SECURE=true
|
||||
GF_USERS_ALLOW_SIGN_UP=false
|
||||
GF_AUTH_ANONYMOUS_ENABLED=false
|
||||
GF_ANALYTICS_REPORTING_ENABLED=false
|
||||
GF_ANALYTICS_CHECK_FOR_UPDATES=false
|
||||
GF_NEWS_NEWS_FEED_ENABLED=false
|
||||
GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/var/lib/grafana/dashboards/overview.json
|
||||
GF_SMTP_ENABLED=true
|
||||
GF_SMTP_HOST=smtp.fastmail.com:587
|
||||
GF_SMTP_USER={{ vault_forgejo_smtp_user }}
|
||||
GF_SMTP_PASSWORD={{ vault_forgejo_smtp_password }}
|
||||
GF_SMTP_FROM_ADDRESS={{ vault_forgejo_smtp_user }}
|
||||
GF_SMTP_FROM_NAME=asxpio monitoring
|
||||
GF_SMTP_STARTTLS_POLICY=MandatoryStartTLS
|
||||
37
ansible/roles/monitoring/templates/loki.yml.j2
Normal file
37
ansible/roles/monitoring/templates/loki.yml.j2
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
# Managed by Ansible (role monitoring).
|
||||
auth_enabled: false
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
log_level: warn
|
||||
common:
|
||||
path_prefix: /loki
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
schema_config:
|
||||
configs:
|
||||
- from: "2026-11-01"
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
limits_config:
|
||||
retention_period: {{ monitoring_logs_retention }}
|
||||
reject_old_samples: true
|
||||
reject_old_samples_max_age: 168h
|
||||
ingestion_rate_mb: 8
|
||||
ingestion_burst_size_mb: 16
|
||||
volume_enabled: true
|
||||
compactor:
|
||||
working_directory: /loki/compactor
|
||||
retention_enabled: true
|
||||
delete_request_store: filesystem
|
||||
analytics:
|
||||
reporting_enabled: false
|
||||
75
ansible/roles/monitoring/templates/prometheus.yml.j2
Normal file
75
ansible/roles/monitoring/templates/prometheus.yml.j2
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
# Managed by Ansible (role monitoring).
|
||||
global:
|
||||
scrape_interval: 30s
|
||||
evaluation_interval: 30s
|
||||
|
||||
scrape_configs:
|
||||
- job_name: node
|
||||
static_configs:
|
||||
{% for h in monitoring_hosts %}
|
||||
- targets: ["{{ h.address }}:{{ monitoring_node_exporter_port }}"]
|
||||
labels: { host: {{ h.name }} }
|
||||
{% endfor %}
|
||||
|
||||
- job_name: cadvisor
|
||||
scrape_interval: 60s
|
||||
static_configs:
|
||||
{% for h in monitoring_hosts %}
|
||||
- targets: ["{{ h.address }}:{{ monitoring_cadvisor_port }}"]
|
||||
labels: { host: {{ h.name }} }
|
||||
{% endfor %}
|
||||
metric_relabel_configs:
|
||||
# Short-lived CI job containers would bloat the series count
|
||||
- source_labels: [name]
|
||||
regex: FORGEJO-ACTIONS-TASK-.*
|
||||
action: drop
|
||||
|
||||
- job_name: alloy
|
||||
static_configs:
|
||||
{% for h in monitoring_hosts %}
|
||||
- targets: ["{{ h.address }}:12345"]
|
||||
labels: { host: {{ h.name }} }
|
||||
{% endfor %}
|
||||
|
||||
- job_name: traefik
|
||||
static_configs:
|
||||
- targets: ["traefik:8082"]
|
||||
labels: { host: asxpio-ops }
|
||||
|
||||
- job_name: forgejo
|
||||
metrics_path: /metrics
|
||||
authorization:
|
||||
credentials: "{{ monitoring_forgejo_metrics_token }}"
|
||||
static_configs:
|
||||
- targets: ["forgejo:3000"]
|
||||
labels: { host: asxpio-ops }
|
||||
|
||||
- job_name: postgres
|
||||
static_configs:
|
||||
- targets: ["postgres-exporter:9187"]
|
||||
labels: { host: asxpio-ops }
|
||||
|
||||
- job_name: monitoring
|
||||
static_configs:
|
||||
- targets: ["localhost:9090", "loki:3100", "grafana:3000", "blackbox:9115"]
|
||||
labels: { host: asxpio-ops }
|
||||
|
||||
{% for module in monitoring_probes | map(attribute='module') | unique %}
|
||||
- job_name: probe_{{ module }}
|
||||
metrics_path: /probe
|
||||
params:
|
||||
module: [{{ module }}]
|
||||
static_configs:
|
||||
{% for p in monitoring_probes if p.module == module %}
|
||||
- targets: ["{{ p.target }}"]
|
||||
labels: { probe: "{{ p.name }}" }
|
||||
{% endfor %}
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: blackbox:9115
|
||||
|
||||
{% endfor %}
|
||||
22
ansible/roles/monitoring_agent/tasks/main.yaml
Normal file
22
ansible/roles/monitoring_agent/tasks/main.yaml
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
---
|
||||
# node_exporter, cAdvisor and Alloy on every host. Metrics listen on the
|
||||
# host's VPC address only; Alloy ships Docker logs and the journal to Loki
|
||||
# on asxpio-ops over the VPC.
|
||||
- name: Create the node_exporter textfile directory
|
||||
ansible.builtin.file:
|
||||
path: "{{ monitoring_textfile_dir }}"
|
||||
state: directory
|
||||
mode: "0755"
|
||||
|
||||
- name: Deploy the monitoring agents
|
||||
ansible.builtin.include_role:
|
||||
name: docker_service
|
||||
vars:
|
||||
service_name: monitoring-agent
|
||||
service_base_path: "{{ services_base_path }}/monitoring-agent"
|
||||
service_templates:
|
||||
- { src: docker-compose.yml.j2, dest: docker-compose.yml }
|
||||
- { src: config.alloy.j2, dest: config.alloy }
|
||||
health_check_enabled: true
|
||||
health_check_host: "{{ vpc_ip }}"
|
||||
health_check_port: "{{ monitoring_node_exporter_port }}"
|
||||
78
ansible/roles/monitoring_agent/templates/config.alloy.j2
Normal file
78
ansible/roles/monitoring_agent/templates/config.alloy.j2
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
// Managed by Ansible (role monitoring_agent): logs of {{ inventory_hostname }} to Loki.
|
||||
|
||||
discovery.docker "containers" {
|
||||
host = "unix:///var/run/docker.sock"
|
||||
}
|
||||
|
||||
discovery.relabel "containers" {
|
||||
targets = discovery.docker.containers.targets
|
||||
|
||||
rule {
|
||||
source_labels = ["__meta_docker_container_name"]
|
||||
regex = "/(.*)"
|
||||
target_label = "container"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_docker_container_label_com_docker_compose_project"]
|
||||
target_label = "compose_project"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
|
||||
target_label = "compose_service"
|
||||
}
|
||||
}
|
||||
|
||||
loki.source.docker "containers" {
|
||||
host = "unix:///var/run/docker.sock"
|
||||
targets = discovery.docker.containers.targets
|
||||
relabel_rules = discovery.relabel.containers.rules
|
||||
labels = { job = "docker" }
|
||||
forward_to = [loki.process.containers.receiver]
|
||||
}
|
||||
|
||||
loki.process "containers" {
|
||||
// CI job containers are many, short-lived and noisy: the runner logs suffice
|
||||
stage.match {
|
||||
selector = "{container=~\"FORGEJO-ACTIONS-TASK-.*\"}"
|
||||
action = "drop"
|
||||
}
|
||||
forward_to = [loki.write.default.receiver]
|
||||
}
|
||||
|
||||
loki.relabel "journal" {
|
||||
forward_to = []
|
||||
|
||||
rule {
|
||||
source_labels = ["__journal__systemd_unit"]
|
||||
target_label = "unit"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__journal_priority_keyword"]
|
||||
target_label = "level"
|
||||
}
|
||||
}
|
||||
|
||||
loki.source.journal "journal" {
|
||||
max_age = "12h"
|
||||
relabel_rules = loki.relabel.journal.rules
|
||||
labels = { job = "journal" }
|
||||
forward_to = [loki.write.default.receiver]
|
||||
}
|
||||
{% if inventory_hostname == 'asxpio-ops' %}
|
||||
|
||||
local.file_match "traefik" {
|
||||
path_targets = [{ "__path__" = "/traefik-logs/access.log", job = "traefik-access" }]
|
||||
}
|
||||
|
||||
loki.source.file "traefik" {
|
||||
targets = local.file_match.traefik.targets
|
||||
forward_to = [loki.write.default.receiver]
|
||||
}
|
||||
{% endif %}
|
||||
|
||||
loki.write "default" {
|
||||
endpoint {
|
||||
url = "http://{{ asxpio_ops_vpc_ip }}:{{ monitoring_loki_port }}/loki/api/v1/push"
|
||||
}
|
||||
external_labels = { host = "{{ inventory_hostname }}" }
|
||||
}
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
# Managed by Ansible (role monitoring_agent).
|
||||
services:
|
||||
node-exporter:
|
||||
image: prom/node-exporter:{{ monitoring_node_exporter_version }}
|
||||
container_name: node-exporter
|
||||
command:
|
||||
- --path.rootfs=/host
|
||||
- --web.listen-address={{ vpc_ip }}:{{ monitoring_node_exporter_port }}
|
||||
- --collector.textfile.directory=/textfile
|
||||
- --collector.filesystem.mount-points-exclude=^/(dev|proc|run|sys|var/lib/docker/.+)($|/)
|
||||
network_mode: host
|
||||
pid: host
|
||||
volumes:
|
||||
- /:/host:ro,rslave
|
||||
- {{ monitoring_textfile_dir }}:/textfile:ro
|
||||
restart: unless-stopped
|
||||
mem_limit: 64m
|
||||
|
||||
cadvisor:
|
||||
image: ghcr.io/google/cadvisor:{{ monitoring_cadvisor_version }}
|
||||
container_name: cadvisor
|
||||
command:
|
||||
- --listen_ip={{ vpc_ip }}
|
||||
- --port={{ monitoring_cadvisor_port }}
|
||||
- --docker_only=true
|
||||
- --housekeeping_interval=30s
|
||||
- --store_container_labels=false
|
||||
- --whitelisted_container_labels=com.docker.compose.project,com.docker.compose.service
|
||||
- --disable_metrics=percpu,sched,tcp,udp,hugetlb,referenced_memory,cpu_topology,resctrl,advtcp,process
|
||||
network_mode: host
|
||||
privileged: true
|
||||
devices:
|
||||
- /dev/kmsg
|
||||
volumes:
|
||||
- /:/rootfs:ro
|
||||
- /var/run:/var/run:ro
|
||||
- /sys:/sys:ro
|
||||
- /var/lib/docker/:/var/lib/docker:ro
|
||||
- /dev/disk/:/dev/disk:ro
|
||||
restart: unless-stopped
|
||||
mem_limit: 192m
|
||||
|
||||
alloy:
|
||||
image: grafana/alloy:{{ monitoring_alloy_version }}
|
||||
container_name: alloy
|
||||
command:
|
||||
- run
|
||||
- --server.http.listen-addr={{ vpc_ip }}:12345
|
||||
- --storage.path=/var/lib/alloy/data
|
||||
- /etc/alloy/config.alloy
|
||||
network_mode: host
|
||||
volumes:
|
||||
- ./config.alloy:/etc/alloy/config.alloy:ro
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
- /var/log/journal:/var/log/journal:ro
|
||||
- /run/log/journal:/run/log/journal:ro
|
||||
- /etc/machine-id:/etc/machine-id:ro
|
||||
{% if inventory_hostname == 'asxpio-ops' %}
|
||||
- /opt/traefik/logs:/traefik-logs:ro
|
||||
{% endif %}
|
||||
- alloy-data:/var/lib/alloy/data
|
||||
restart: unless-stopped
|
||||
mem_limit: 256m
|
||||
|
||||
volumes:
|
||||
alloy-data:
|
||||
|
|
@ -22,6 +22,12 @@ declare -A REPOS=(
|
|||
|
||||
log() { echo "restic-backup: $*"; }
|
||||
|
||||
# Last good backup per repository, for Prometheus (node_exporter textfile)
|
||||
metric() {
|
||||
local f={{ monitoring_textfile_dir }}/restic_$1.prom
|
||||
printf '# HELP restic_backup_last_success_timestamp_seconds Last successful restic backup.\n# TYPE restic_backup_last_success_timestamp_seconds gauge\nrestic_backup_last_success_timestamp_seconds{repo="%s"} %s\n' "$1" "$(date +%s)" > "$f.tmp" && mv "$f.tmp" "$f"
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
init)
|
||||
for name in "${!REPOS[@]}"; do
|
||||
|
|
@ -56,6 +62,7 @@ run)
|
|||
[ "$(date +%a)" = "{{ restic_prune_weekday }}" ] && prune="--prune"
|
||||
log "$name: forget $prune"
|
||||
restic forget --host {{ inventory_hostname }} {{ restic_keep }} $prune || failed=1
|
||||
metric "$name"
|
||||
done
|
||||
|
||||
rm -rf "$STAGE"
|
||||
|
|
|
|||
|
|
@ -16,4 +16,8 @@ rclone sync hel1:{{ p.src }} fsn1:{{ p.dst }} \
|
|||
{% endfor %}
|
||||
--stats-one-line -v || failed=1
|
||||
{% endfor %}
|
||||
if [ $failed = 0 ]; then
|
||||
f={{ monitoring_textfile_dir }}/s3_replicate.prom
|
||||
printf '# HELP s3_replicate_last_success_timestamp_seconds Last fully successful hel1 to fsn1 replica.\n# TYPE s3_replicate_last_success_timestamp_seconds gauge\ns3_replicate_last_success_timestamp_seconds %s\n' "$(date +%s)" > "$f.tmp" && mv "$f.tmp" "$f"
|
||||
fi
|
||||
exit $failed
|
||||
|
|
|
|||
|
|
@ -16,6 +16,15 @@ entryPoints:
|
|||
scheme: https
|
||||
websecure:
|
||||
address: ":443"
|
||||
# Prometheus metrics, reached on the traefik network only (not published)
|
||||
metrics:
|
||||
address: ":8082"
|
||||
|
||||
metrics:
|
||||
prometheus:
|
||||
entryPoint: metrics
|
||||
addRoutersLabels: true
|
||||
addServicesLabels: true
|
||||
|
||||
# Docker Provider
|
||||
providers:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue