crawler_alert, monitoring: AI crawler metrics and dashboards
Some checks failed
deploy / ansible (push) Failing after 1m17s

crawler-alert.py --metrics (5-minute timer) reads what the Traefik
access log gained since the last run, judges every request that names
an AI bot exactly like the weekly report (vendor IP ranges, forward-
confirmed rDNS, Meta's ASN) and writes cumulative counters to the
node_exporter textfile directory. Verdicts are cached per bot and IP
for a week, so DNS only sees new addresses; range files are refetched
once a day. It covers every router, while the mails keep watching
Forgejo only.

New dashboard "AI crawlers", and a row of it on the overview.

The overview's load gauge divided node_load1 by a count with fewer
labels, which matches nothing; aggregate both sides by host.

Assisted-by: Claude:opus-5-5
This commit is contained in:
Sergei Poljanski 2026-11-13 07:21:57 +04:00
commit bc0dac9188
Signed by: asxpi
GPG key ID: 4F8851660FA4121B
7 changed files with 1924 additions and 6 deletions

View file

@ -286,6 +286,177 @@ def fcrdns(ip, suffixes):
_rdns_cache[key] = result _rdns_cache[key] = result
return result return result
# --- --metrics: Prometheus counters for Grafana --------------------------
# Run every few minutes from a timer. Reads only what the live log gained
# since the last run (byte offset + inode in a state file, so rotation and
# truncation restart at 0), classifies each request the same way as the
# weekly report, and writes cumulative counters to node_exporter's
# textfile directory. Verdicts per (bot, IP) are cached for a week, so DNS
# is only asked about addresses it hasn't seen.
METRICS = "--metrics" in sys.argv
TEXTFILE = os.environ.get("CRAWLER_TEXTFILE", "/var/lib/node_exporter/textfile/crawler.prom")
STATE = os.environ.get("CRAWLER_STATE", "/var/cache/crawler-alert/metrics.json")
LIVE_LOG = os.environ.get("CRAWLER_LIVE_LOG", sorted(glob.glob(LOG_GLOB) or [""])[0])
ALL_BOTS = sorted(set(BOT_VENDOR) | set(BOT_RDNS) | set(BOT_ASN), key=len, reverse=True)
GENERIC_BOT = re.compile(r"bot|crawler|spider|scrapy|externalagent", re.I)
UA_RE = re.compile(r"\"[^\"]*\" \"([^\"]*)\" \d+ \"")
# The mail reports watch one router (Forgejo); the dashboard wants them all.
METRICS_ROUTER = os.environ.get("CRAWLER_METRICS_ROUTER", ROUTER)
def vendor_of(bot):
if bot in BOT_VENDOR:
return BOT_VENDOR[bot]
if bot in BOT_ASN:
return "meta"
return bot.lower()
def index_ranges(vendors, cf):
idx = {}
for vendor, nets in vendors.items():
b = {}
for s in nets:
n = ipaddress.ip_network(s)
b.setdefault(int(n.network_address) >> 24 if n.version == 4 else -1, []).append(n)
idx[vendor] = b
return idx, [ipaddress.ip_network(s) for s in cf]
def cached_ranges(max_age=86400):
"""Range files change rarely: refetch once a day, not every run."""
import time
try:
if time.time() - os.path.getmtime(CACHE) < max_age:
c = json.load(open(CACHE))
return index_ranges(c["vendors"], c.get("cloudflare", []))
except Exception:
pass
idx, cf, _stale = load_ranges()
return idx, cf
def judge(bot, ip, ranges, cfnets, cache, budget):
"""verified | spoofed | proxied | private | unknown, like the report:
IP ranges first, then forward-confirmed rDNS, then single-tenant ASN."""
try:
addr = ipaddress.ip_address(ip)
except ValueError:
return "unknown"
if addr.is_private:
return "private"
if any(addr in n for n in cfnets):
return "proxied"
if bot in BOT_VENDOR and in_index(ranges.get(BOT_VENDOR[bot]), addr):
return "verified"
if bot not in BOT_RDNS and bot not in BOT_ASN:
return "spoofed" # range-only vendor, address outside ranges
key = f"{bot}|{ip}"
if key in cache:
return cache[key][0]
if budget[0] <= 0:
return "unknown" # over this run's DNS budget; retried next run
budget[0] -= 1
v = "unknown"
ok, why = False, "no-ptr"
if bot in BOT_RDNS:
ok, _host, why = fcrdns(ip, BOT_RDNS[bot])
if ok:
v = "verified"
elif why == "no-ptr":
if ASN_ENABLE and bot in BOT_ASN and asn_of(ip) in BOT_ASN[bot]:
v = "verified"
else:
v = "spoofed" # a PTR exists and points elsewhere
cache[key] = [v, int(date.today().toordinal())]
return v
def run_metrics():
try:
st = json.load(open(STATE))
except Exception:
st = {}
counts = Counter(st.get("counts", {}))
cache = {k: v for k, v in st.get("verdicts", {}).items()
if v[1] > date.today().toordinal() - 7}
try:
fs = os.stat(LIVE_LOG)
except OSError:
sys.exit(f"cannot read {LIVE_LOG}")
if "inode" not in st:
offset = fs.st_size # first run: start at the end, not the backlog
elif st["inode"] != fs.st_ino or fs.st_size < st.get("offset", 0):
offset = 0 # rotated or truncated
else:
offset = st["offset"]
ranges, cfnets = cached_ranges()
socket.setdefaulttimeout(3)
budget = [RDNS_MAX]
with open(LIVE_LOG, "rb") as f:
f.seek(offset)
for raw in f:
if not raw.endswith(b"\n"):
break # half-written line: take it next run
offset += len(raw)
line = raw.decode("utf-8", "replace")
if METRICS_ROUTER and METRICS_ROUTER not in line:
continue
parts = line.split()
counts["requests"] += 1
m = UA_RE.search(line)
u = m.group(1) if m else "-"
bot = next((b for b in ALL_BOTS if b in u), None)
if bot:
v = judge(bot, parts[0], ranges, cfnets, cache, budget)
counts[f"ai|{vendor_of(bot)}|{bot}|{v}"] += 1
if v == "verified" and len(parts) > 9 and parts[9].isdigit():
counts[f"bytes|{vendor_of(bot)}"] += int(parts[9])
elif u == "-":
counts["class|none"] += 1
elif GENERIC_BOT.search(u):
counts["class|other-bot"] += 1
else:
counts["class|browser-like"] += 1
if len(cache) > 50000: # keep the newest verdicts only
cache = dict(sorted(cache.items(), key=lambda kv: kv[1][1])[-50000:])
os.makedirs(os.path.dirname(STATE), exist_ok=True)
json.dump({"inode": fs.st_ino, "offset": offset, "counts": counts,
"verdicts": cache}, open(STATE + ".tmp", "w"))
os.replace(STATE + ".tmp", STATE)
out = ["# HELP crawler_requests_total Requests seen in the Traefik access log.",
"# TYPE crawler_requests_total counter",
f"crawler_requests_total {counts['requests']}",
"# HELP crawler_class_requests_total Requests without an AI-bot user agent, by user-agent class.",
"# TYPE crawler_class_requests_total counter"]
for k, c in sorted(counts.items()):
if k.startswith("class|"):
out.append(f'crawler_class_requests_total{{class="{k[6:]}"}} {c}')
out += ["# HELP crawler_ai_requests_total Requests claiming an AI-bot user agent, by verification verdict.",
"# TYPE crawler_ai_requests_total counter"]
for k, c in sorted(counts.items()):
if k.startswith("ai|"):
_, vendor, bot, v = k.split("|")
out.append(f'crawler_ai_requests_total{{vendor="{vendor}",bot="{bot}",verdict="{v}"}} {c}')
out += ["# HELP crawler_ai_verified_bytes_total Bytes served to verified AI crawlers.",
"# TYPE crawler_ai_verified_bytes_total counter"]
for k, c in sorted(counts.items()):
if k.startswith("bytes|"):
out.append(f'crawler_ai_verified_bytes_total{{vendor="{k[6:]}"}} {c}')
import time
out += ["# HELP crawler_metrics_last_run_timestamp_seconds Last successful --metrics run.",
"# TYPE crawler_metrics_last_run_timestamp_seconds gauge",
f"crawler_metrics_last_run_timestamp_seconds {int(time.time())}"]
with open(TEXTFILE + ".tmp", "w") as f:
f.write("\n".join(out) + "\n")
os.replace(TEXTFILE + ".tmp", TEXTFILE)
if METRICS:
run_metrics()
sys.exit(0)
days = [date.today() - timedelta(days=i) for i in range(7 if WEEKLY else 1, 0, -1)] days = [date.today() - timedelta(days=i) for i in range(7 if WEEKLY else 1, 0, -1)]
stamps = [d.strftime("%d/%b/%Y") for d in days] stamps = [d.strftime("%d/%b/%Y") for d in days]

View file

@ -0,0 +1,12 @@
[Unit]
Description=Crawler counters for Prometheus (node_exporter textfile)
After=docker.service
[Service]
Type=oneshot
EnvironmentFile=/etc/crawler-alert.env
# Every router, not only Forgejo's; fewer DNS lookups per 5-minute run
Environment=CRAWLER_METRICS_ROUTER= CRAWLER_RDNS_MAX=100
ExecStart=/usr/local/bin/crawler-alert.py --metrics
Nice=10
TimeoutStartSec=4min

View file

@ -0,0 +1,9 @@
[Unit]
Description=Crawler counters every 5 minutes
[Timer]
OnCalendar=*:0/5
RandomizedDelaySec=30
[Install]
WantedBy=timers.target

View file

@ -22,6 +22,8 @@
- crawler-alert.timer - crawler-alert.timer
- crawler-weekly.service - crawler-weekly.service
- crawler-weekly.timer - crawler-weekly.timer
- crawler-metrics.service
- crawler-metrics.timer
notify: Reload systemd notify: Reload systemd
- name: Flush handlers - name: Flush handlers
@ -35,3 +37,4 @@
loop: loop:
- crawler-alert.timer - crawler-alert.timer
- crawler-weekly.timer - crawler-weekly.timer
- crawler-metrics.timer

View file

@ -6,3 +6,5 @@ CRAWLER_THRESHOLD={{ crawler_threshold }}
CRAWLER_MAIL_CONTAINER=forgejo CRAWLER_MAIL_CONTAINER=forgejo
CRAWLER_MAIL_INI=/data/gitea/conf/app.ini CRAWLER_MAIL_INI=/data/gitea/conf/app.ini
CRAWLER_ROBOTS={{ crawler_robots_url }} CRAWLER_ROBOTS={{ crawler_robots_url }}
CRAWLER_LIVE_LOG=/opt/traefik/logs/access.log
CRAWLER_TEXTFILE={{ monitoring_textfile_dir }}/crawler.prom

File diff suppressed because it is too large Load diff

View file

@ -403,7 +403,7 @@
"type": "prometheus", "type": "prometheus",
"uid": "prometheus" "uid": "prometheus"
}, },
"expr": "node_load1 / count by (host) (node_cpu_seconds_total{mode=\"idle\"})", "expr": "sum by (host) (node_load1) / count by (host) (node_cpu_seconds_total{mode=\"idle\"})",
"legendFormat": "{{host}}", "legendFormat": "{{host}}",
"refId": "A", "refId": "A",
"instant": true, "instant": true,
@ -769,7 +769,7 @@
"uid": "prometheus" "uid": "prometheus"
}, },
"expr": "topk(12, container_memory_working_set_bytes{name!=\"\"})", "expr": "topk(12, container_memory_working_set_bytes{name!=\"\"})",
"legendFormat": "{{host}} \u00b7 {{name}}", "legendFormat": "{{host}} · {{name}}",
"refId": "A", "refId": "A",
"instant": true, "instant": true,
"range": false "range": false
@ -826,7 +826,7 @@
"uid": "prometheus" "uid": "prometheus"
}, },
"expr": "topk(8, sum by (host, name) (rate(container_cpu_usage_seconds_total{name!=\"\"}[$__rate_interval])))", "expr": "topk(8, sum by (host, name) (rate(container_cpu_usage_seconds_total{name!=\"\"}[$__rate_interval])))",
"legendFormat": "{{host}} \u00b7 {{name}}", "legendFormat": "{{host}} · {{name}}",
"refId": "A", "refId": "A",
"instant": false, "instant": false,
"range": true "range": true
@ -1321,7 +1321,7 @@
"uid": "prometheus" "uid": "prometheus"
}, },
"expr": "(time() - restic_backup_last_success_timestamp_seconds) / 3600", "expr": "(time() - restic_backup_last_success_timestamp_seconds) / 3600",
"legendFormat": "restic \u00b7 {{host}} \u00b7 {{repo}}", "legendFormat": "restic · {{host}} · {{repo}}",
"refId": "A", "refId": "A",
"instant": true, "instant": true,
"range": false "range": false
@ -1332,7 +1332,7 @@
"uid": "prometheus" "uid": "prometheus"
}, },
"expr": "(time() - s3_replicate_last_success_timestamp_seconds) / 3600", "expr": "(time() - s3_replicate_last_success_timestamp_seconds) / 3600",
"legendFormat": "replica hel1 \u2192 fsn1", "legendFormat": "replica hel1 → fsn1",
"refId": "B", "refId": "B",
"instant": true, "instant": true,
"range": false "range": false
@ -1488,6 +1488,579 @@
"enableLogDetails": true, "enableLogDetails": true,
"dedupStrategy": "signature" "dedupStrategy": "signature"
} }
},
{
"type": "row",
"title": "AI crawlers",
"id": 30,
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 59
},
"panels": []
},
{
"type": "piechart",
"title": "Who is asking (time range)",
"id": 31,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "sum(increase(crawler_ai_requests_total{verdict=\"verified\"}[$__range]))",
"legendFormat": "AI · verified",
"refId": "A",
"instant": true,
"range": false
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "sum(increase(crawler_ai_requests_total{verdict=\"spoofed\"}[$__range]))",
"legendFormat": "AI · spoofed",
"refId": "B",
"instant": true,
"range": false
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "sum(increase(crawler_ai_requests_total{verdict=~\"unknown|proxied|private\"}[$__range]))",
"legendFormat": "AI · unverifiable",
"refId": "C",
"instant": true,
"range": false
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "sum(increase(crawler_class_requests_total{class=\"other-bot\"}[$__range]))",
"legendFormat": "other bots",
"refId": "D",
"instant": true,
"range": false
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "sum(increase(crawler_class_requests_total{class=\"browser-like\"}[$__range]))",
"legendFormat": "browser-like",
"refId": "E",
"instant": true,
"range": false
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "sum(increase(crawler_class_requests_total{class=\"none\"}[$__range]))",
"legendFormat": "no user agent",
"refId": "F",
"instant": true,
"range": false
}
],
"gridPos": {
"h": 8,
"w": 6,
"x": 0,
"y": 60
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "short"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "verified"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "green"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "spoofed"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "red"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "proxied"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "blue"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "unknown"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "yellow"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "private"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "purple"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "AI · verified"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "green"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "AI · spoofed"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "red"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "AI · unverifiable"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "yellow"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "other bots"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "orange"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "browser-like"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "purple"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "no user agent"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "text"
}
}
]
}
]
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"pieType": "donut",
"legend": {
"displayMode": "table",
"placement": "right",
"showLegend": true,
"values": [
"value",
"percent"
]
},
"tooltip": {
"mode": "single"
}
},
"description": "From the Traefik access log, every router (crawler-alert.py --metrics, 5-minute timer). A request naming an AI bot counts as verified only if its source IP is in the vendor's published ranges, or passes forward-confirmed reverse DNS, or (Meta) sits in the vendor's own ASN. spoofed = the claim failed that check; unverifiable = via Cloudflare, no PTR, or over the DNS budget.",
"links": [
{
"title": "Crawler details",
"url": "/d/asxpio-crawlers?${__url_time_range}"
}
]
},
{
"type": "timeseries",
"title": "AI crawler requests by verdict (per minute)",
"id": 32,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "sum by (verdict) (rate(crawler_ai_requests_total[$__rate_interval])) * 60",
"legendFormat": "{{verdict}}",
"refId": "A",
"instant": false,
"range": true
}
],
"gridPos": {
"h": 8,
"w": 9,
"x": 6,
"y": 60
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "short",
"custom": {
"stacking": {
"mode": "normal"
},
"fillOpacity": 40,
"lineWidth": 1
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "verified"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "green"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "spoofed"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "red"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "proxied"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "blue"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "unknown"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "yellow"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "private"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "purple"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "AI · verified"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "green"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "AI · spoofed"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "red"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "AI · unverifiable"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "yellow"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "other bots"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "orange"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "browser-like"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "purple"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "no user agent"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "text"
}
}
]
}
]
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"description": "From the Traefik access log, every router (crawler-alert.py --metrics, 5-minute timer). A request naming an AI bot counts as verified only if its source IP is in the vendor's published ranges, or passes forward-confirmed reverse DNS, or (Meta) sits in the vendor's own ASN. spoofed = the claim failed that check; unverifiable = via Cloudflare, no PTR, or over the DNS budget."
},
{
"type": "bargauge",
"title": "Verified AI crawlers by vendor (time range)",
"id": 33,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "sort_desc(sum by (vendor) (increase(crawler_ai_requests_total{verdict=\"verified\"}[$__range])))",
"legendFormat": "{{vendor}}",
"refId": "A",
"instant": true,
"range": false
}
],
"gridPos": {
"h": 8,
"w": 9,
"x": 15,
"y": 60
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "continuous-GrYlRd"
},
"unit": "short",
"decimals": 0,
"min": 0
},
"overrides": []
},
"options": {
"displayMode": "gradient",
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"values": false
},
"showUnfilled": true,
"valueMode": "color"
}
} }
], ],
"graphTooltip": 1, "graphTooltip": 1,
@ -1501,4 +2074,4 @@
"tags": [] "tags": []
} }
] ]
} }