ansible: monitoring with Prometheus, Loki and Grafana
All checks were successful
deploy / ansible (push) Successful in 7m23s
All checks were successful
deploy / ansible (push) Successful in 7m23s
Nothing watched the servers: tonight's asxpio crash loop was found by hand. On asxpio-ops, Prometheus (30 days), Loki (14 days) and Grafana at gf.asxp.io (WireGuard only), with blackbox_exporter probing every service from outside and postgres_exporter. On both hosts node_exporter and cAdvisor on the VPC address, and Alloy shipping container logs, the journal and Traefik's access log to Loki over the VPC. Traefik and Forgejo expose their metrics; restic and the bucket replica record their last success for node_exporter. Dashboards: an overview as home, plus hosts, containers, Traefik, probes, Postgres and logs. Alerts by email to me@asxp.io: host or service down, certificate under 14 days, disk or memory nearly full, a container restart-looping, Postgres down, backup or replica stale. Assisted-by: Claude:opus-5-5
This commit is contained in:
parent
5289542dca
commit
e20f5a7d07
GPG key ID:
4F8851660FA4121B
28 changed files with 24705 additions and 161 deletions
24
ansible/roles/monitoring/defaults/main.yaml
Normal file
24
ansible/roles/monitoring/defaults/main.yaml
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
---
|
||||
monitoring_base_path: "{{ services_base_path }}/monitoring"
|
||||
|
||||
# Hosts with node_exporter, cAdvisor and Alloy, scraped over the VPC
|
||||
monitoring_hosts:
|
||||
- { name: asxpio-ops, address: "{{ asxpio_ops_vpc_ip }}" }
|
||||
- { name: asxpio-services, address: "{{ asxpio_services_vpc_ip }}" }
|
||||
|
||||
# Outside-in probes (blackbox_exporter). module: see blackbox.yml.j2
|
||||
monitoring_probes:
|
||||
- { name: asxp.io, target: "https://asxp.io/", module: http_2xx }
|
||||
- { name: blog, target: "https://blog.asxp.io/", module: http_2xx }
|
||||
- { name: czsk.it, target: "https://czsk.it/", module: http_2xx }
|
||||
- { name: datacenterbingo, target: "https://datacenterbingo.xyz/", module: http_2xx }
|
||||
- { name: datacenterbingo data.json, target: "https://datacenterbingo.xyz/data.json", module: http_2xx }
|
||||
- { name: forgejo, target: "https://git.asxp.io/", module: http_2xx }
|
||||
- { name: s3, target: "https://s3.asxp.io/", module: http_up }
|
||||
- { name: sillytavern, target: "http://{{ asxpio_services_wireguard_ip }}:8000/", module: http_up }
|
||||
- { name: xmpp c2s starttls, target: "xmpp.asxp.io:5222", module: xmpp_starttls }
|
||||
- { name: xmpp c2s tls, target: "xmpp.asxp.io:5223", module: tls_connect }
|
||||
- { name: xmpp s2s, target: "xmpp.asxp.io:5269", module: tcp_connect }
|
||||
- { name: turn, target: "xmpp.asxp.io:3478", module: tcp_connect }
|
||||
- { name: xray, target: "{{ hostvars['asxpio-services'].public_ip }}:443", module: tcp_connect }
|
||||
- { name: minecraft, target: "mc.asxp.io:25565", module: tcp_connect }
|
||||
|
|
@ -0,0 +1,781 @@
|
|||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": "-- Grafana --",
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"gnetId": 14282,
|
||||
"graphTooltip": 0,
|
||||
"id": null,
|
||||
"iteration": 1617715580880,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"collapsed": false,
|
||||
"datasource": "prometheus",
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 8,
|
||||
"panels": [],
|
||||
"title": "CPU",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 1
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 15,
|
||||
"legend": {
|
||||
"alignAsTable": true,
|
||||
"avg": true,
|
||||
"current": false,
|
||||
"max": true,
|
||||
"min": false,
|
||||
"rightSide": true,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": true
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"nullPointMode": "null as zero",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "7.4.5",
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": true,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(container_cpu_usage_seconds_total{instance=~\"$host\",name=~\"$container\",name=~\".+\"}[5m])) by (name) *100",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "CPU Usage",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"$$hashKey": "object:606",
|
||||
"format": "percent",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"$$hashKey": "object:607",
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"datasource": "prometheus",
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 8
|
||||
},
|
||||
"id": 11,
|
||||
"panels": [],
|
||||
"title": "Memory",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 9
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 9,
|
||||
"legend": {
|
||||
"alignAsTable": true,
|
||||
"avg": true,
|
||||
"current": false,
|
||||
"max": true,
|
||||
"min": false,
|
||||
"rightSide": true,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": true
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"nullPointMode": "null as zero",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "7.4.5",
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": true,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(container_memory_rss{instance=~\"$host\",name=~\"$container\",name=~\".+\"}) by (name)",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "Memory Usage",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"$$hashKey": "object:606",
|
||||
"format": "bytes",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"$$hashKey": "object:607",
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 9
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 14,
|
||||
"legend": {
|
||||
"alignAsTable": true,
|
||||
"avg": true,
|
||||
"current": false,
|
||||
"max": true,
|
||||
"min": false,
|
||||
"rightSide": true,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": true
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"nullPointMode": "null as zero",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "7.4.5",
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": true,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(container_memory_cache{instance=~\"$host\",name=~\"$container\",name=~\".+\"}) by (name)",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "Memory Cached",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"$$hashKey": "object:606",
|
||||
"format": "bytes",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"$$hashKey": "object:607",
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"datasource": "prometheus",
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 17
|
||||
},
|
||||
"id": 2,
|
||||
"panels": [],
|
||||
"title": "Network",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 18
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 4,
|
||||
"legend": {
|
||||
"alignAsTable": true,
|
||||
"avg": true,
|
||||
"current": false,
|
||||
"hideEmpty": false,
|
||||
"hideZero": false,
|
||||
"max": true,
|
||||
"min": false,
|
||||
"rightSide": true,
|
||||
"show": true,
|
||||
"sideWidth": null,
|
||||
"total": false,
|
||||
"values": true
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"nullPointMode": "null",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "7.4.5",
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(container_network_receive_bytes_total{instance=~\"$host\",name=~\"$container\",name=~\".+\"}[5m])) by (name)",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "Received Network Traffic",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"$$hashKey": "object:674",
|
||||
"format": "Bps",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"$$hashKey": "object:675",
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 18
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 6,
|
||||
"legend": {
|
||||
"alignAsTable": true,
|
||||
"avg": true,
|
||||
"current": false,
|
||||
"max": true,
|
||||
"min": false,
|
||||
"rightSide": true,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": true
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"nullPointMode": "null",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "7.4.5",
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(container_network_transmit_bytes_total{instance=~\"$host\",name=~\"$container\",name=~\".+\"}[5m])) by (name)",
|
||||
"interval": "",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "Sent Network Traffic",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"$$hashKey": "object:832",
|
||||
"format": "Bps",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"$$hashKey": "object:833",
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"datasource": "prometheus",
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 26
|
||||
},
|
||||
"id": 19,
|
||||
"panels": [],
|
||||
"title": "Misc",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": "prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"align": null,
|
||||
"filterable": false
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "id"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "custom.width",
|
||||
"value": 260
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "Running"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "unit",
|
||||
"value": "d"
|
||||
},
|
||||
{
|
||||
"id": "decimals",
|
||||
"value": 1
|
||||
},
|
||||
{
|
||||
"id": "custom.displayMode",
|
||||
"value": "color-text"
|
||||
},
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"fixedColor": "dark-green",
|
||||
"mode": "fixed"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 10,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 27
|
||||
},
|
||||
"id": 17,
|
||||
"options": {
|
||||
"showHeader": true,
|
||||
"sortBy": []
|
||||
},
|
||||
"pluginVersion": "7.4.5",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(time() - container_start_time_seconds{instance=~\"$host\",name=~\"$container\",name=~\".+\"})/86400",
|
||||
"format": "table",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"timeFrom": null,
|
||||
"timeShift": null,
|
||||
"title": "Containers Info",
|
||||
"transformations": [
|
||||
{
|
||||
"id": "filterFieldsByName",
|
||||
"options": {
|
||||
"include": {
|
||||
"names": [
|
||||
"container_label_com_docker_compose_project",
|
||||
"container_label_com_docker_compose_project_working_dir",
|
||||
"image",
|
||||
"instance",
|
||||
"name",
|
||||
"Value",
|
||||
"container_label_com_docker_compose_service"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "organize",
|
||||
"options": {
|
||||
"excludeByName": {},
|
||||
"indexByName": {},
|
||||
"renameByName": {
|
||||
"Value": "Running",
|
||||
"container_label_com_docker_compose_project": "Label",
|
||||
"container_label_com_docker_compose_project_working_dir": "Working dir",
|
||||
"container_label_com_docker_compose_service": "Service",
|
||||
"image": "Registry Image",
|
||||
"instance": "Instance",
|
||||
"name": "Name"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"type": "table"
|
||||
}
|
||||
],
|
||||
"schemaVersion": 27,
|
||||
"style": "dark",
|
||||
"tags": [
|
||||
"cadvisor",
|
||||
"docker"
|
||||
],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"allValue": ".*",
|
||||
"current": {},
|
||||
"datasource": "prometheus",
|
||||
"definition": "label_values({__name__=~\"container.*\"},instance)",
|
||||
"description": null,
|
||||
"error": null,
|
||||
"hide": 0,
|
||||
"includeAll": true,
|
||||
"label": "Host",
|
||||
"multi": false,
|
||||
"name": "host",
|
||||
"options": [],
|
||||
"query": {
|
||||
"query": "label_values({__name__=~\"container.*\"},instance)",
|
||||
"refId": "Prometheus-host-Variable-Query"
|
||||
},
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
"sort": 5,
|
||||
"tagValuesQuery": "",
|
||||
"tags": [],
|
||||
"tagsQuery": "",
|
||||
"type": "query",
|
||||
"useTags": false
|
||||
},
|
||||
{
|
||||
"allValue": ".*",
|
||||
"current": {},
|
||||
"datasource": "prometheus",
|
||||
"definition": "label_values({__name__=~\"container.*\", instance=~\"$host\"},name)",
|
||||
"description": null,
|
||||
"error": null,
|
||||
"hide": 0,
|
||||
"includeAll": true,
|
||||
"label": "Container",
|
||||
"multi": false,
|
||||
"name": "container",
|
||||
"options": [],
|
||||
"query": {
|
||||
"query": "label_values({__name__=~\"container.*\", instance=~\"$host\"},name)",
|
||||
"refId": "Prometheus-container-Variable-Query"
|
||||
},
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
"sort": 0,
|
||||
"tagValuesQuery": "",
|
||||
"tags": [],
|
||||
"tagsQuery": "",
|
||||
"type": "query",
|
||||
"useTags": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {
|
||||
"from": "now-6h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "",
|
||||
"title": "Cadvisor exporter",
|
||||
"uid": "gnet-14282",
|
||||
"version": 1,
|
||||
"description": "Simple exporter for cadvisor only"
|
||||
}
|
||||
15517
ansible/roles/monitoring/files/dashboards/Infrastructure/hosts.json
Normal file
15517
ansible/roles/monitoring/files/dashboards/Infrastructure/hosts.json
Normal file
File diff suppressed because it is too large
Load diff
248
ansible/roles/monitoring/files/dashboards/Logs/logs.json
Normal file
248
ansible/roles/monitoring/files/dashboards/Logs/logs.json
Normal file
|
|
@ -0,0 +1,248 @@
|
|||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"$$hashKey": "object:75",
|
||||
"builtIn": 1,
|
||||
"datasource": "-- Grafana --",
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"description": "Log Viewer Dashboard for Loki",
|
||||
"editable": false,
|
||||
"gnetId": 13639,
|
||||
"graphTooltip": 0,
|
||||
"id": null,
|
||||
"iteration": 1608932746420,
|
||||
"links": [
|
||||
{
|
||||
"$$hashKey": "object:59",
|
||||
"icon": "bolt",
|
||||
"includeVars": true,
|
||||
"keepTime": true,
|
||||
"tags": [],
|
||||
"targetBlank": true,
|
||||
"title": "View In Explore",
|
||||
"type": "link",
|
||||
"url": "/explore?orgId=1&left=[\"now-1h\",\"now\",\"Loki\",{\"expr\":\"{job=\\\"$app\\\"}\"},{\"ui\":[true,true,true,\"none\"]}]"
|
||||
},
|
||||
{
|
||||
"$$hashKey": "object:61",
|
||||
"icon": "external link",
|
||||
"tags": [],
|
||||
"targetBlank": true,
|
||||
"title": "Learn LogQL",
|
||||
"type": "link",
|
||||
"url": "https://grafana.com/docs/loki/latest/logql/"
|
||||
}
|
||||
],
|
||||
"panels": [
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": true,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "loki",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {},
|
||||
"links": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"gridPos": {
|
||||
"h": 3,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 6,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": false,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": false,
|
||||
"linewidth": 1,
|
||||
"nullPointMode": "null",
|
||||
"percentage": false,
|
||||
"pluginVersion": "7.1.0",
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(count_over_time({job=\"$app\"} |= \"$search\" [$__interval]))",
|
||||
"legendFormat": "",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"$$hashKey": "object:168",
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": false
|
||||
},
|
||||
{
|
||||
"$$hashKey": "object:169",
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": false
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"datasource": "loki",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 25,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 3
|
||||
},
|
||||
"id": 2,
|
||||
"maxDataPoints": "",
|
||||
"options": {
|
||||
"showLabels": false,
|
||||
"showTime": true,
|
||||
"sortOrder": "Descending",
|
||||
"wrapLogMessage": false
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "{job=\"$app\"} |= \"$search\" | logfmt",
|
||||
"hide": false,
|
||||
"legendFormat": "",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"timeFrom": null,
|
||||
"timeShift": null,
|
||||
"title": "",
|
||||
"transparent": true,
|
||||
"type": "logs"
|
||||
}
|
||||
],
|
||||
"refresh": false,
|
||||
"schemaVersion": 26,
|
||||
"style": "dark",
|
||||
"tags": [],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"allValue": null,
|
||||
"current": {},
|
||||
"datasource": "loki",
|
||||
"definition": "label_values(job)",
|
||||
"hide": 0,
|
||||
"includeAll": false,
|
||||
"label": "App",
|
||||
"multi": false,
|
||||
"name": "app",
|
||||
"options": [],
|
||||
"query": "label_values(job)",
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
"sort": 0,
|
||||
"tagValuesQuery": "",
|
||||
"tags": [],
|
||||
"tagsQuery": "",
|
||||
"type": "query",
|
||||
"useTags": false
|
||||
},
|
||||
{
|
||||
"current": {
|
||||
"selected": false,
|
||||
"text": "",
|
||||
"value": ""
|
||||
},
|
||||
"hide": 0,
|
||||
"label": "String Match",
|
||||
"name": "search",
|
||||
"options": [
|
||||
{
|
||||
"selected": true,
|
||||
"text": "",
|
||||
"value": ""
|
||||
}
|
||||
],
|
||||
"query": "",
|
||||
"skipUrlSync": false,
|
||||
"type": "textbox"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {
|
||||
"hidden": false,
|
||||
"refresh_intervals": [
|
||||
"10s",
|
||||
"30s",
|
||||
"1m",
|
||||
"5m",
|
||||
"15m",
|
||||
"30m",
|
||||
"1h",
|
||||
"2h",
|
||||
"1d"
|
||||
]
|
||||
},
|
||||
"timezone": "",
|
||||
"title": "Logs / App",
|
||||
"uid": "gnet-13639",
|
||||
"version": 13
|
||||
}
|
||||
3131
ansible/roles/monitoring/files/dashboards/Services/postgres.json
Normal file
3131
ansible/roles/monitoring/files/dashboards/Services/postgres.json
Normal file
File diff suppressed because it is too large
Load diff
1071
ansible/roles/monitoring/files/dashboards/Services/probes.json
Normal file
1071
ansible/roles/monitoring/files/dashboards/Services/probes.json
Normal file
File diff suppressed because it is too large
Load diff
1527
ansible/roles/monitoring/files/dashboards/Services/traefik.json
Normal file
1527
ansible/roles/monitoring/files/dashboards/Services/traefik.json
Normal file
File diff suppressed because it is too large
Load diff
1504
ansible/roles/monitoring/files/dashboards/overview.json
Normal file
1504
ansible/roles/monitoring/files/dashboards/overview.json
Normal file
File diff suppressed because it is too large
Load diff
37
ansible/roles/monitoring/tasks/main.yaml
Normal file
37
ansible/roles/monitoring/tasks/main.yaml
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
---
|
||||
# Prometheus, Loki, Grafana, blackbox_exporter and postgres_exporter on
|
||||
# asxpio-ops. Grafana: https://gf.asxp.io, WireGuard only.
|
||||
- name: Create the provisioning directories
|
||||
ansible.builtin.file:
|
||||
path: "{{ monitoring_base_path }}/{{ item }}"
|
||||
state: directory
|
||||
mode: "0755"
|
||||
loop:
|
||||
- grafana/provisioning/datasources
|
||||
- grafana/provisioning/dashboards
|
||||
- grafana/provisioning/alerting
|
||||
- grafana/dashboards
|
||||
|
||||
- name: Deploy the dashboards
|
||||
ansible.builtin.copy:
|
||||
src: dashboards/
|
||||
dest: "{{ monitoring_base_path }}/grafana/dashboards/"
|
||||
mode: "0644"
|
||||
|
||||
- name: Deploy the monitoring stack
|
||||
ansible.builtin.include_role:
|
||||
name: docker_service
|
||||
vars:
|
||||
service_name: monitoring
|
||||
service_base_path: "{{ monitoring_base_path }}"
|
||||
service_templates:
|
||||
- { src: docker-compose.yml.j2, dest: docker-compose.yml }
|
||||
- { src: prometheus.yml.j2, dest: prometheus.yml }
|
||||
- { src: blackbox.yml.j2, dest: blackbox.yml }
|
||||
- { src: loki.yml.j2, dest: loki.yml }
|
||||
- { src: grafana.env.j2, dest: grafana.env, mode: "0600" }
|
||||
- { src: datasources.yaml.j2, dest: grafana/provisioning/datasources/datasources.yaml }
|
||||
- { src: dashboards.yaml.j2, dest: grafana/provisioning/dashboards/dashboards.yaml }
|
||||
- { src: alerting.yaml.j2, dest: grafana/provisioning/alerting/alerting.yaml }
|
||||
- { src: alert-rules.yaml.j2, dest: grafana/provisioning/alerting/rules.yaml }
|
||||
health_check_url: "http://{{ asxpio_ops_vpc_ip }}:{{ monitoring_loki_port }}/ready"
|
||||
170
ansible/roles/monitoring/templates/alert-rules.yaml.j2
Normal file
170
ansible/roles/monitoring/templates/alert-rules.yaml.j2
Normal file
|
|
@ -0,0 +1,170 @@
|
|||
# Managed by Ansible (role monitoring). Generated; see the commit adding it.
|
||||
{% raw %}
|
||||
apiVersion: 1
|
||||
groups:
|
||||
- orgId: 1
|
||||
name: asxpio
|
||||
folder: Alerts
|
||||
interval: 1m
|
||||
rules:
|
||||
- uid: host-down
|
||||
title: "Host down"
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 900, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model: { refId: A, expr: "1 - up{job=\"node\"}", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [0] } } ] }
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 2m
|
||||
labels: { severity: critical }
|
||||
annotations: { summary: "{{ $labels.host }} stopped answering (node_exporter over the VPC)" }
|
||||
- uid: target-down
|
||||
title: "Scrape target down"
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 900, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model: { refId: A, expr: "up{job!~\"node|probe_.*\"} == 0", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [-1] } } ] }
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 5m
|
||||
labels: { severity: warning }
|
||||
annotations: { summary: "{{ $labels.job }} on {{ $labels.instance }} is not answering metrics scrapes" }
|
||||
- uid: probe-down
|
||||
title: "Service down (outside-in probe)"
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 900, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model: { refId: A, expr: "1 - probe_success", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [0] } } ] }
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 2m
|
||||
labels: { severity: critical }
|
||||
annotations: { summary: "{{ $labels.probe }} ({{ $labels.instance }}) is not reachable" }
|
||||
- uid: cert-expiry
|
||||
title: "Certificate expires soon"
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 900, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model: { refId: A, expr: "(probe_ssl_earliest_cert_expiry - time()) / 86400", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: lt, params: [14] } } ] }
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 1h
|
||||
labels: { severity: warning }
|
||||
annotations: { summary: "{{ $labels.probe }}: certificate expires in {{ $values.A.Value | printf \"%.0f\" }} days" }
|
||||
- uid: disk-full
|
||||
title: "Disk almost full"
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 900, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model: { refId: A, expr: "100 * (1 - node_filesystem_avail_bytes{fstype!~\"tmpfs|overlay|squashfs|fuse.*\"} / node_filesystem_size_bytes{fstype!~\"tmpfs|overlay|squashfs|fuse.*\"})", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [85] } } ] }
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 10m
|
||||
labels: { severity: warning }
|
||||
annotations: { summary: "{{ $labels.host }} {{ $labels.mountpoint }} is {{ $values.A.Value | printf \"%.0f\" }}% full" }
|
||||
- uid: memory-low
|
||||
title: "Memory almost exhausted"
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 900, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model: { refId: A, expr: "100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [92] } } ] }
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 10m
|
||||
labels: { severity: warning }
|
||||
annotations: { summary: "{{ $labels.host }} memory is {{ $values.A.Value | printf \"%.0f\" }}% used" }
|
||||
- uid: container-restarting
|
||||
title: "Container restart-looping"
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 900, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model: { refId: A, expr: "changes(container_start_time_seconds{name!=\"\"}[15m])", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [2] } } ] }
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 0m
|
||||
labels: { severity: critical }
|
||||
annotations: { summary: "{{ $labels.name }} on {{ $labels.host }} restarted {{ $values.A.Value | printf \"%.0f\" }} times in 15 minutes" }
|
||||
- uid: postgres-down
|
||||
title: "Postgres down"
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 900, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model: { refId: A, expr: "pg_up", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: lt, params: [1] } } ] }
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 1m
|
||||
labels: { severity: critical }
|
||||
annotations: { summary: "Postgres (storage) is not answering" }
|
||||
- uid: backup-stale
|
||||
title: "Backup stale"
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 900, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model: { refId: A, expr: "(time() - restic_backup_last_success_timestamp_seconds) / 3600", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [26] } } ] }
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 0m
|
||||
labels: { severity: warning }
|
||||
annotations: { summary: "restic on {{ $labels.host }}: last good backup {{ $values.A.Value | printf \"%.0f\" }} hours ago" }
|
||||
- uid: replica-stale
|
||||
title: "Bucket replica stale"
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 900, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model: { refId: A, expr: "(time() - s3_replicate_last_success_timestamp_seconds) / 3600", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [26] } } ] }
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 0m
|
||||
labels: { severity: warning }
|
||||
annotations: { summary: "hel1 -> fsn1 bucket replica: last good run {{ $values.A.Value | printf \"%.0f\" }} hours ago" }
|
||||
{% endraw %}
|
||||
17
ansible/roles/monitoring/templates/alerting.yaml.j2
Normal file
17
ansible/roles/monitoring/templates/alerting.yaml.j2
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
apiVersion: 1
|
||||
contactPoints:
|
||||
- orgId: 1
|
||||
name: email
|
||||
receivers:
|
||||
- uid: email-me
|
||||
type: email
|
||||
settings:
|
||||
addresses: {{ monitoring_alert_email }}
|
||||
singleEmail: true
|
||||
policies:
|
||||
- orgId: 1
|
||||
receiver: email
|
||||
group_by: [alertname, host]
|
||||
group_wait: 1m
|
||||
group_interval: 5m
|
||||
repeat_interval: 4h
|
||||
38
ansible/roles/monitoring/templates/blackbox.yml.j2
Normal file
38
ansible/roles/monitoring/templates/blackbox.yml.j2
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
# Managed by Ansible (role monitoring).
|
||||
modules:
|
||||
http_2xx:
|
||||
prober: http
|
||||
timeout: 10s
|
||||
http:
|
||||
preferred_ip_protocol: ip4
|
||||
follow_redirects: true
|
||||
# Up means "answers": 401/403/404 are fine (s3 root, allowlisted UIs)
|
||||
http_up:
|
||||
prober: http
|
||||
timeout: 10s
|
||||
http:
|
||||
preferred_ip_protocol: ip4
|
||||
valid_status_codes: [200, 301, 302, 401, 403, 404]
|
||||
tls_connect:
|
||||
prober: tcp
|
||||
timeout: 10s
|
||||
tcp:
|
||||
preferred_ip_protocol: ip4
|
||||
tls: true
|
||||
tcp_connect:
|
||||
prober: tcp
|
||||
timeout: 10s
|
||||
tcp:
|
||||
preferred_ip_protocol: ip4
|
||||
# XMPP client port: stream header, STARTTLS, then the certificate
|
||||
xmpp_starttls:
|
||||
prober: tcp
|
||||
timeout: 10s
|
||||
tcp:
|
||||
preferred_ip_protocol: ip4
|
||||
query_response:
|
||||
- send: "<?xml version='1.0'?><stream:stream to='asxp.io' xmlns='jabber:client' xmlns:stream='http://etherx.jabber.org/streams' version='1.0'>"
|
||||
- expect: "urn:ietf:params:xml:ns:xmpp-tls"
|
||||
- send: "<starttls xmlns='urn:ietf:params:xml:ns:xmpp-tls'/>"
|
||||
- expect: "<proceed"
|
||||
- starttls: true
|
||||
9
ansible/roles/monitoring/templates/dashboards.yaml.j2
Normal file
9
ansible/roles/monitoring/templates/dashboards.yaml.j2
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
apiVersion: 1
|
||||
providers:
|
||||
- name: asxpio
|
||||
type: file
|
||||
disableDeletion: true
|
||||
allowUiUpdates: false
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
foldersFromFilesStructure: true
|
||||
14
ansible/roles/monitoring/templates/datasources.yaml.j2
Normal file
14
ansible/roles/monitoring/templates/datasources.yaml.j2
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
apiVersion: 1
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
jsonData: { timeInterval: 30s }
|
||||
- name: Loki
|
||||
uid: loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
87
ansible/roles/monitoring/templates/docker-compose.yml.j2
Normal file
87
ansible/roles/monitoring/templates/docker-compose.yml.j2
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
# Managed by Ansible (role monitoring).
|
||||
services:
|
||||
prometheus:
|
||||
image: prom/prometheus:{{ monitoring_prometheus_version }}
|
||||
container_name: prometheus
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time={{ monitoring_metrics_retention }}
|
||||
- --web.enable-lifecycle
|
||||
volumes:
|
||||
- ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- prometheus-data:/prometheus
|
||||
networks: [monitoring, traefik, storage]
|
||||
restart: unless-stopped
|
||||
mem_limit: 640m
|
||||
|
||||
loki:
|
||||
image: grafana/loki:{{ monitoring_loki_version }}
|
||||
container_name: loki
|
||||
command: ["-config.file=/etc/loki/loki.yml"]
|
||||
ports:
|
||||
# Alloy on both hosts pushes here over the VPC
|
||||
- "{{ asxpio_ops_vpc_ip }}:{{ monitoring_loki_port }}:3100"
|
||||
volumes:
|
||||
- ./loki.yml:/etc/loki/loki.yml:ro
|
||||
- loki-data:/loki
|
||||
networks: [monitoring]
|
||||
restart: unless-stopped
|
||||
mem_limit: 512m
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:{{ monitoring_grafana_version }}
|
||||
container_name: grafana
|
||||
env_file: [grafana.env]
|
||||
volumes:
|
||||
- grafana-data:/var/lib/grafana
|
||||
- ./grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
networks: [monitoring, traefik]
|
||||
restart: unless-stopped
|
||||
mem_limit: 384m
|
||||
labels:
|
||||
- traefik.enable=true
|
||||
- traefik.docker.network=traefik
|
||||
- traefik.http.routers.grafana.rule=Host(`{{ monitoring_grafana_host }}`)
|
||||
- traefik.http.routers.grafana.entrypoints=websecure
|
||||
- traefik.http.routers.grafana.tls=true
|
||||
- traefik.http.routers.grafana.tls.certresolver=letsencrypt
|
||||
- traefik.http.routers.grafana.middlewares=grafana-wireguard-only
|
||||
- traefik.http.middlewares.grafana-wireguard-only.ipallowlist.sourcerange={{ wireguard_network_cidr }}
|
||||
- traefik.http.services.grafana.loadbalancer.server.port=3000
|
||||
|
||||
blackbox:
|
||||
image: prom/blackbox-exporter:{{ monitoring_blackbox_version }}
|
||||
container_name: blackbox
|
||||
command: ["--config.file=/etc/blackbox/blackbox.yml"]
|
||||
volumes:
|
||||
- ./blackbox.yml:/etc/blackbox/blackbox.yml:ro
|
||||
networks: [monitoring]
|
||||
restart: unless-stopped
|
||||
mem_limit: 64m
|
||||
|
||||
postgres-exporter:
|
||||
image: quay.io/prometheuscommunity/postgres-exporter:{{ monitoring_postgres_exporter_version }}
|
||||
container_name: postgres-exporter
|
||||
# The password comes from the storage stack's own .env (same host)
|
||||
env_file: [{{ services_base_path }}/storage/.env]
|
||||
environment:
|
||||
DATA_SOURCE_URI: storage-postgres:5432/postgres?sslmode=disable
|
||||
DATA_SOURCE_USER: postgres
|
||||
entrypoint: ["/bin/sh", "-c", "DATA_SOURCE_PASS=\"$$POSTGRES_PASSWORD\" exec /bin/postgres_exporter"]
|
||||
networks: [monitoring, storage]
|
||||
restart: unless-stopped
|
||||
mem_limit: 64m
|
||||
|
||||
volumes:
|
||||
prometheus-data:
|
||||
loki-data:
|
||||
grafana-data:
|
||||
|
||||
networks:
|
||||
monitoring:
|
||||
traefik:
|
||||
external: true
|
||||
storage:
|
||||
external: true
|
||||
17
ansible/roles/monitoring/templates/grafana.env.j2
Normal file
17
ansible/roles/monitoring/templates/grafana.env.j2
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
GF_SERVER_ROOT_URL=https://{{ monitoring_grafana_host }}/
|
||||
GF_SECURITY_ADMIN_USER=admin
|
||||
GF_SECURITY_ADMIN_PASSWORD={{ monitoring_grafana_admin_password }}
|
||||
GF_SECURITY_COOKIE_SECURE=true
|
||||
GF_USERS_ALLOW_SIGN_UP=false
|
||||
GF_AUTH_ANONYMOUS_ENABLED=false
|
||||
GF_ANALYTICS_REPORTING_ENABLED=false
|
||||
GF_ANALYTICS_CHECK_FOR_UPDATES=false
|
||||
GF_NEWS_NEWS_FEED_ENABLED=false
|
||||
GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/var/lib/grafana/dashboards/overview.json
|
||||
GF_SMTP_ENABLED=true
|
||||
GF_SMTP_HOST=smtp.fastmail.com:587
|
||||
GF_SMTP_USER={{ vault_forgejo_smtp_user }}
|
||||
GF_SMTP_PASSWORD={{ vault_forgejo_smtp_password }}
|
||||
GF_SMTP_FROM_ADDRESS={{ vault_forgejo_smtp_user }}
|
||||
GF_SMTP_FROM_NAME=asxpio monitoring
|
||||
GF_SMTP_STARTTLS_POLICY=MandatoryStartTLS
|
||||
37
ansible/roles/monitoring/templates/loki.yml.j2
Normal file
37
ansible/roles/monitoring/templates/loki.yml.j2
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
# Managed by Ansible (role monitoring).
|
||||
auth_enabled: false
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
log_level: warn
|
||||
common:
|
||||
path_prefix: /loki
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
schema_config:
|
||||
configs:
|
||||
- from: "2026-11-01"
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
limits_config:
|
||||
retention_period: {{ monitoring_logs_retention }}
|
||||
reject_old_samples: true
|
||||
reject_old_samples_max_age: 168h
|
||||
ingestion_rate_mb: 8
|
||||
ingestion_burst_size_mb: 16
|
||||
volume_enabled: true
|
||||
compactor:
|
||||
working_directory: /loki/compactor
|
||||
retention_enabled: true
|
||||
delete_request_store: filesystem
|
||||
analytics:
|
||||
reporting_enabled: false
|
||||
75
ansible/roles/monitoring/templates/prometheus.yml.j2
Normal file
75
ansible/roles/monitoring/templates/prometheus.yml.j2
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
# Managed by Ansible (role monitoring).
|
||||
global:
|
||||
scrape_interval: 30s
|
||||
evaluation_interval: 30s
|
||||
|
||||
scrape_configs:
|
||||
- job_name: node
|
||||
static_configs:
|
||||
{% for h in monitoring_hosts %}
|
||||
- targets: ["{{ h.address }}:{{ monitoring_node_exporter_port }}"]
|
||||
labels: { host: {{ h.name }} }
|
||||
{% endfor %}
|
||||
|
||||
- job_name: cadvisor
|
||||
scrape_interval: 60s
|
||||
static_configs:
|
||||
{% for h in monitoring_hosts %}
|
||||
- targets: ["{{ h.address }}:{{ monitoring_cadvisor_port }}"]
|
||||
labels: { host: {{ h.name }} }
|
||||
{% endfor %}
|
||||
metric_relabel_configs:
|
||||
# Short-lived CI job containers would bloat the series count
|
||||
- source_labels: [name]
|
||||
regex: FORGEJO-ACTIONS-TASK-.*
|
||||
action: drop
|
||||
|
||||
- job_name: alloy
|
||||
static_configs:
|
||||
{% for h in monitoring_hosts %}
|
||||
- targets: ["{{ h.address }}:12345"]
|
||||
labels: { host: {{ h.name }} }
|
||||
{% endfor %}
|
||||
|
||||
- job_name: traefik
|
||||
static_configs:
|
||||
- targets: ["traefik:8082"]
|
||||
labels: { host: asxpio-ops }
|
||||
|
||||
- job_name: forgejo
|
||||
metrics_path: /metrics
|
||||
authorization:
|
||||
credentials: "{{ monitoring_forgejo_metrics_token }}"
|
||||
static_configs:
|
||||
- targets: ["forgejo:3000"]
|
||||
labels: { host: asxpio-ops }
|
||||
|
||||
- job_name: postgres
|
||||
static_configs:
|
||||
- targets: ["postgres-exporter:9187"]
|
||||
labels: { host: asxpio-ops }
|
||||
|
||||
- job_name: monitoring
|
||||
static_configs:
|
||||
- targets: ["localhost:9090", "loki:3100", "grafana:3000", "blackbox:9115"]
|
||||
labels: { host: asxpio-ops }
|
||||
|
||||
{% for module in monitoring_probes | map(attribute='module') | unique %}
|
||||
- job_name: probe_{{ module }}
|
||||
metrics_path: /probe
|
||||
params:
|
||||
module: [{{ module }}]
|
||||
static_configs:
|
||||
{% for p in monitoring_probes if p.module == module %}
|
||||
- targets: ["{{ p.target }}"]
|
||||
labels: { probe: "{{ p.name }}" }
|
||||
{% endfor %}
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: blackbox:9115
|
||||
|
||||
{% endfor %}
|
||||
Loading…
Add table
Add a link
Reference in a new issue