Sergei Poljanski
bc0dac9188
Some checks failed
deploy / ansible (push) Failing after 1m17s
crawler-alert.py --metrics (5-minute timer) reads what the Traefik access log gained since the last run, judges every request that names an AI bot exactly like the weekly report (vendor IP ranges, forward- confirmed rDNS, Meta's ASN) and writes cumulative counters to the node_exporter textfile directory. Verdicts are cached per bot and IP for a week, so DNS only sees new addresses; range files are refetched once a day. It covers every router, while the mails keep watching Forgejo only. New dashboard "AI crawlers", and a row of it on the overview. The overview's load gauge divided node_load1 by a count with fewer labels, which matches nothing; aggregate both sides by host. Assisted-by: Claude:opus-5-5
1148 lines
21 KiB
JSON
1148 lines
21 KiB
JSON
{
|
|
"uid": "asxpio-crawlers",
|
|
"title": "AI crawlers",
|
|
"tags": [
|
|
"asxpio",
|
|
"traefik"
|
|
],
|
|
"timezone": "browser",
|
|
"editable": false,
|
|
"refresh": "5m",
|
|
"time": {
|
|
"from": "now-24h",
|
|
"to": "now"
|
|
},
|
|
"schemaVersion": 39,
|
|
"graphTooltip": 1,
|
|
"links": [
|
|
{
|
|
"title": "Dashboards",
|
|
"type": "dashboards",
|
|
"asDropdown": true,
|
|
"includeVars": false,
|
|
"keepTime": true,
|
|
"tags": []
|
|
}
|
|
],
|
|
"panels": [
|
|
{
|
|
"type": "piechart",
|
|
"title": "Who is asking (time range)",
|
|
"id": 1,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum(increase(crawler_ai_requests_total{verdict=\"verified\"}[$__range]))",
|
|
"legendFormat": "AI · verified",
|
|
"refId": "A",
|
|
"instant": true,
|
|
"range": false
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum(increase(crawler_ai_requests_total{verdict=\"spoofed\"}[$__range]))",
|
|
"legendFormat": "AI · spoofed",
|
|
"refId": "B",
|
|
"instant": true,
|
|
"range": false
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum(increase(crawler_ai_requests_total{verdict=~\"unknown|proxied|private\"}[$__range]))",
|
|
"legendFormat": "AI · unverifiable",
|
|
"refId": "C",
|
|
"instant": true,
|
|
"range": false
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum(increase(crawler_class_requests_total{class=\"other-bot\"}[$__range]))",
|
|
"legendFormat": "other bots",
|
|
"refId": "D",
|
|
"instant": true,
|
|
"range": false
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum(increase(crawler_class_requests_total{class=\"browser-like\"}[$__range]))",
|
|
"legendFormat": "browser-like",
|
|
"refId": "E",
|
|
"instant": true,
|
|
"range": false
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum(increase(crawler_class_requests_total{class=\"none\"}[$__range]))",
|
|
"legendFormat": "no user agent",
|
|
"refId": "F",
|
|
"instant": true,
|
|
"range": false
|
|
}
|
|
],
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 6,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "verified"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "green"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "spoofed"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "red"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "proxied"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "blue"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "unknown"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "yellow"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "private"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "purple"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "AI · verified"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "green"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "AI · spoofed"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "red"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "AI · unverifiable"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "yellow"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "other bots"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "orange"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "browser-like"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "purple"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "no user agent"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "text"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"pieType": "donut",
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "right",
|
|
"showLegend": true,
|
|
"values": [
|
|
"value",
|
|
"percent"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "single"
|
|
}
|
|
},
|
|
"description": "From the Traefik access log, every router (crawler-alert.py --metrics, 5-minute timer). A request naming an AI bot counts as verified only if its source IP is in the vendor's published ranges, or passes forward-confirmed reverse DNS, or (Meta) sits in the vendor's own ASN. spoofed = the claim failed that check; unverifiable = via Cloudflare, no PTR, or over the DNS budget."
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "AI crawler requests by verdict (per minute)",
|
|
"id": 2,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum by (verdict) (rate(crawler_ai_requests_total[$__rate_interval])) * 60",
|
|
"legendFormat": "{{verdict}}",
|
|
"refId": "A",
|
|
"instant": false,
|
|
"range": true
|
|
}
|
|
],
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 9,
|
|
"x": 6,
|
|
"y": 0
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"unit": "short",
|
|
"custom": {
|
|
"stacking": {
|
|
"mode": "normal"
|
|
},
|
|
"fillOpacity": 40,
|
|
"lineWidth": 1
|
|
}
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "verified"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "green"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "spoofed"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "red"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "proxied"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "blue"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "unknown"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "yellow"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "private"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "purple"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "AI · verified"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "green"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "AI · spoofed"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "red"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "AI · unverifiable"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "yellow"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "other bots"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "orange"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "browser-like"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "purple"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "no user agent"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "text"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"description": "From the Traefik access log, every router (crawler-alert.py --metrics, 5-minute timer). A request naming an AI bot counts as verified only if its source IP is in the vendor's published ranges, or passes forward-confirmed reverse DNS, or (Meta) sits in the vendor's own ASN. spoofed = the claim failed that check; unverifiable = via Cloudflare, no PTR, or over the DNS budget."
|
|
},
|
|
{
|
|
"type": "bargauge",
|
|
"title": "Verified AI crawlers by vendor (time range)",
|
|
"id": 3,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sort_desc(sum by (vendor) (increase(crawler_ai_requests_total{verdict=\"verified\"}[$__range])))",
|
|
"legendFormat": "{{vendor}}",
|
|
"refId": "A",
|
|
"instant": true,
|
|
"range": false
|
|
}
|
|
],
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 9,
|
|
"x": 15,
|
|
"y": 0
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "continuous-GrYlRd"
|
|
},
|
|
"unit": "short",
|
|
"decimals": 0,
|
|
"min": 0
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"displayMode": "gradient",
|
|
"orientation": "horizontal",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"values": false
|
|
},
|
|
"showUnfilled": true,
|
|
"valueMode": "color"
|
|
}
|
|
},
|
|
{
|
|
"type": "stat",
|
|
"title": "Spoofed share of AI claims",
|
|
"id": 4,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum(increase(crawler_ai_requests_total{verdict=\"spoofed\"}[$__range])) / sum(increase(crawler_ai_requests_total[$__range]))",
|
|
"legendFormat": "spoofed",
|
|
"refId": "A",
|
|
"instant": true,
|
|
"range": false
|
|
}
|
|
],
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 0,
|
|
"y": 8
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit",
|
|
"decimals": 1,
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "orange",
|
|
"value": 0.25
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 0.5
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "none",
|
|
"textMode": "value_and_name",
|
|
"justifyMode": "center",
|
|
"orientation": "auto"
|
|
}
|
|
},
|
|
{
|
|
"type": "stat",
|
|
"title": "Served to verified AI crawlers",
|
|
"id": 5,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum(increase(crawler_ai_verified_bytes_total[$__range]))",
|
|
"legendFormat": "bytes",
|
|
"refId": "A",
|
|
"instant": true,
|
|
"range": false
|
|
}
|
|
],
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 6,
|
|
"y": 8
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "bytes",
|
|
"color": {
|
|
"mode": "fixed",
|
|
"fixedColor": "green"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "none",
|
|
"textMode": "value_and_name",
|
|
"justifyMode": "center",
|
|
"orientation": "auto"
|
|
}
|
|
},
|
|
{
|
|
"type": "stat",
|
|
"title": "All requests (time range)",
|
|
"id": 6,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum(increase(crawler_requests_total[$__range]))",
|
|
"legendFormat": "requests",
|
|
"refId": "A",
|
|
"instant": true,
|
|
"range": false
|
|
}
|
|
],
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 12,
|
|
"y": 8
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 0,
|
|
"color": {
|
|
"mode": "fixed",
|
|
"fixedColor": "text"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "none",
|
|
"textMode": "value_and_name",
|
|
"justifyMode": "center",
|
|
"orientation": "auto"
|
|
}
|
|
},
|
|
{
|
|
"type": "stat",
|
|
"title": "Collector last ran",
|
|
"id": 7,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "time() - crawler_metrics_last_run_timestamp_seconds",
|
|
"legendFormat": "ago",
|
|
"refId": "A",
|
|
"instant": true,
|
|
"range": false
|
|
}
|
|
],
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 18,
|
|
"y": 8
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s",
|
|
"decimals": 0,
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "orange",
|
|
"value": 900
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 3600
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "none",
|
|
"textMode": "value_and_name",
|
|
"justifyMode": "center",
|
|
"orientation": "auto"
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "Verified AI requests by vendor (per minute)",
|
|
"id": 8,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum by (vendor) (rate(crawler_ai_requests_total{verdict=\"verified\"}[$__rate_interval])) * 60",
|
|
"legendFormat": "{{vendor}}",
|
|
"refId": "A",
|
|
"instant": false,
|
|
"range": true
|
|
}
|
|
],
|
|
"gridPos": {
|
|
"h": 9,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 13
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"unit": "short",
|
|
"custom": {
|
|
"stacking": {
|
|
"mode": "normal"
|
|
},
|
|
"fillOpacity": 40,
|
|
"lineWidth": 1
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "Client classes (per minute)",
|
|
"id": 9,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum(rate(crawler_ai_requests_total[$__rate_interval])) * 60",
|
|
"legendFormat": "AI-bot claims",
|
|
"refId": "A",
|
|
"instant": false,
|
|
"range": true
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum by (class) (rate(crawler_class_requests_total[$__rate_interval])) * 60",
|
|
"legendFormat": "{{class}}",
|
|
"refId": "B",
|
|
"instant": false,
|
|
"range": true
|
|
}
|
|
],
|
|
"gridPos": {
|
|
"h": 9,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 13
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"unit": "short",
|
|
"custom": {
|
|
"stacking": {
|
|
"mode": "normal"
|
|
},
|
|
"fillOpacity": 40,
|
|
"lineWidth": 1
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "table",
|
|
"title": "AI bots by verdict (time range)",
|
|
"id": 10,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum by (vendor, bot, verdict) (increase(crawler_ai_requests_total[$__range]))",
|
|
"legendFormat": "",
|
|
"refId": "A",
|
|
"instant": true,
|
|
"range": false
|
|
}
|
|
],
|
|
"gridPos": {
|
|
"h": 11,
|
|
"w": 14,
|
|
"x": 0,
|
|
"y": 22
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 0
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "verified"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.cellOptions",
|
|
"value": {
|
|
"type": "color-text"
|
|
}
|
|
},
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "green"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "spoofed"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.cellOptions",
|
|
"value": {
|
|
"type": "color-text"
|
|
}
|
|
},
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "red"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "unknown"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.cellOptions",
|
|
"value": {
|
|
"type": "color-text"
|
|
}
|
|
},
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "yellow"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "proxied"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.cellOptions",
|
|
"value": {
|
|
"type": "color-text"
|
|
}
|
|
},
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "blue"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"options": {
|
|
"showHeader": true,
|
|
"sortBy": [
|
|
{
|
|
"displayName": "verified",
|
|
"desc": true
|
|
}
|
|
],
|
|
"cellHeight": "sm"
|
|
},
|
|
"transformations": [
|
|
{
|
|
"id": "labelsToFields",
|
|
"options": {}
|
|
},
|
|
{
|
|
"id": "groupingToMatrix",
|
|
"options": {
|
|
"columnField": "verdict",
|
|
"rowField": "bot",
|
|
"valueField": "Value"
|
|
}
|
|
},
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"renameByName": {
|
|
"bot\\verdict": "bot"
|
|
}
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"type": "bargauge",
|
|
"title": "Bandwidth to verified AI crawlers (time range)",
|
|
"id": 11,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sort_desc(sum by (vendor) (increase(crawler_ai_verified_bytes_total[$__range])))",
|
|
"legendFormat": "{{vendor}}",
|
|
"refId": "A",
|
|
"instant": true,
|
|
"range": false
|
|
}
|
|
],
|
|
"gridPos": {
|
|
"h": 11,
|
|
"w": 10,
|
|
"x": 14,
|
|
"y": 22
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "bytes",
|
|
"color": {
|
|
"mode": "continuous-BlPu"
|
|
},
|
|
"min": 0
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"displayMode": "gradient",
|
|
"orientation": "horizontal",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"values": false
|
|
},
|
|
"showUnfilled": true
|
|
}
|
|
}
|
|
]
|
|
}
|