diff --git a/ansible/roles/crawler_alert/files/crawler-alert.py b/ansible/roles/crawler_alert/files/crawler-alert.py index fa6e84b..5bca6a8 100644 --- a/ansible/roles/crawler_alert/files/crawler-alert.py +++ b/ansible/roles/crawler_alert/files/crawler-alert.py @@ -286,6 +286,177 @@ def fcrdns(ip, suffixes): _rdns_cache[key] = result return result +# --- --metrics: Prometheus counters for Grafana -------------------------- +# Run every few minutes from a timer. Reads only what the live log gained +# since the last run (byte offset + inode in a state file, so rotation and +# truncation restart at 0), classifies each request the same way as the +# weekly report, and writes cumulative counters to node_exporter's +# textfile directory. Verdicts per (bot, IP) are cached for a week, so DNS +# is only asked about addresses it hasn't seen. +METRICS = "--metrics" in sys.argv +TEXTFILE = os.environ.get("CRAWLER_TEXTFILE", "/var/lib/node_exporter/textfile/crawler.prom") +STATE = os.environ.get("CRAWLER_STATE", "/var/cache/crawler-alert/metrics.json") +LIVE_LOG = os.environ.get("CRAWLER_LIVE_LOG", sorted(glob.glob(LOG_GLOB) or [""])[0]) +ALL_BOTS = sorted(set(BOT_VENDOR) | set(BOT_RDNS) | set(BOT_ASN), key=len, reverse=True) +GENERIC_BOT = re.compile(r"bot|crawler|spider|scrapy|externalagent", re.I) +UA_RE = re.compile(r"\"[^\"]*\" \"([^\"]*)\" \d+ \"") +# The mail reports watch one router (Forgejo); the dashboard wants them all. +METRICS_ROUTER = os.environ.get("CRAWLER_METRICS_ROUTER", ROUTER) + + +def vendor_of(bot): + if bot in BOT_VENDOR: + return BOT_VENDOR[bot] + if bot in BOT_ASN: + return "meta" + return bot.lower() + + +def index_ranges(vendors, cf): + idx = {} + for vendor, nets in vendors.items(): + b = {} + for s in nets: + n = ipaddress.ip_network(s) + b.setdefault(int(n.network_address) >> 24 if n.version == 4 else -1, []).append(n) + idx[vendor] = b + return idx, [ipaddress.ip_network(s) for s in cf] + + +def cached_ranges(max_age=86400): + """Range files change rarely: refetch once a day, not every run.""" + import time + try: + if time.time() - os.path.getmtime(CACHE) < max_age: + c = json.load(open(CACHE)) + return index_ranges(c["vendors"], c.get("cloudflare", [])) + except Exception: + pass + idx, cf, _stale = load_ranges() + return idx, cf + + +def judge(bot, ip, ranges, cfnets, cache, budget): + """verified | spoofed | proxied | private | unknown, like the report: + IP ranges first, then forward-confirmed rDNS, then single-tenant ASN.""" + try: + addr = ipaddress.ip_address(ip) + except ValueError: + return "unknown" + if addr.is_private: + return "private" + if any(addr in n for n in cfnets): + return "proxied" + if bot in BOT_VENDOR and in_index(ranges.get(BOT_VENDOR[bot]), addr): + return "verified" + if bot not in BOT_RDNS and bot not in BOT_ASN: + return "spoofed" # range-only vendor, address outside ranges + key = f"{bot}|{ip}" + if key in cache: + return cache[key][0] + if budget[0] <= 0: + return "unknown" # over this run's DNS budget; retried next run + budget[0] -= 1 + v = "unknown" + ok, why = False, "no-ptr" + if bot in BOT_RDNS: + ok, _host, why = fcrdns(ip, BOT_RDNS[bot]) + if ok: + v = "verified" + elif why == "no-ptr": + if ASN_ENABLE and bot in BOT_ASN and asn_of(ip) in BOT_ASN[bot]: + v = "verified" + else: + v = "spoofed" # a PTR exists and points elsewhere + cache[key] = [v, int(date.today().toordinal())] + return v + + +def run_metrics(): + try: + st = json.load(open(STATE)) + except Exception: + st = {} + counts = Counter(st.get("counts", {})) + cache = {k: v for k, v in st.get("verdicts", {}).items() + if v[1] > date.today().toordinal() - 7} + try: + fs = os.stat(LIVE_LOG) + except OSError: + sys.exit(f"cannot read {LIVE_LOG}") + if "inode" not in st: + offset = fs.st_size # first run: start at the end, not the backlog + elif st["inode"] != fs.st_ino or fs.st_size < st.get("offset", 0): + offset = 0 # rotated or truncated + else: + offset = st["offset"] + ranges, cfnets = cached_ranges() + socket.setdefaulttimeout(3) + budget = [RDNS_MAX] + with open(LIVE_LOG, "rb") as f: + f.seek(offset) + for raw in f: + if not raw.endswith(b"\n"): + break # half-written line: take it next run + offset += len(raw) + line = raw.decode("utf-8", "replace") + if METRICS_ROUTER and METRICS_ROUTER not in line: + continue + parts = line.split() + counts["requests"] += 1 + m = UA_RE.search(line) + u = m.group(1) if m else "-" + bot = next((b for b in ALL_BOTS if b in u), None) + if bot: + v = judge(bot, parts[0], ranges, cfnets, cache, budget) + counts[f"ai|{vendor_of(bot)}|{bot}|{v}"] += 1 + if v == "verified" and len(parts) > 9 and parts[9].isdigit(): + counts[f"bytes|{vendor_of(bot)}"] += int(parts[9]) + elif u == "-": + counts["class|none"] += 1 + elif GENERIC_BOT.search(u): + counts["class|other-bot"] += 1 + else: + counts["class|browser-like"] += 1 + if len(cache) > 50000: # keep the newest verdicts only + cache = dict(sorted(cache.items(), key=lambda kv: kv[1][1])[-50000:]) + os.makedirs(os.path.dirname(STATE), exist_ok=True) + json.dump({"inode": fs.st_ino, "offset": offset, "counts": counts, + "verdicts": cache}, open(STATE + ".tmp", "w")) + os.replace(STATE + ".tmp", STATE) + + out = ["# HELP crawler_requests_total Requests seen in the Traefik access log.", + "# TYPE crawler_requests_total counter", + f"crawler_requests_total {counts['requests']}", + "# HELP crawler_class_requests_total Requests without an AI-bot user agent, by user-agent class.", + "# TYPE crawler_class_requests_total counter"] + for k, c in sorted(counts.items()): + if k.startswith("class|"): + out.append(f'crawler_class_requests_total{{class="{k[6:]}"}} {c}') + out += ["# HELP crawler_ai_requests_total Requests claiming an AI-bot user agent, by verification verdict.", + "# TYPE crawler_ai_requests_total counter"] + for k, c in sorted(counts.items()): + if k.startswith("ai|"): + _, vendor, bot, v = k.split("|") + out.append(f'crawler_ai_requests_total{{vendor="{vendor}",bot="{bot}",verdict="{v}"}} {c}') + out += ["# HELP crawler_ai_verified_bytes_total Bytes served to verified AI crawlers.", + "# TYPE crawler_ai_verified_bytes_total counter"] + for k, c in sorted(counts.items()): + if k.startswith("bytes|"): + out.append(f'crawler_ai_verified_bytes_total{{vendor="{k[6:]}"}} {c}') + import time + out += ["# HELP crawler_metrics_last_run_timestamp_seconds Last successful --metrics run.", + "# TYPE crawler_metrics_last_run_timestamp_seconds gauge", + f"crawler_metrics_last_run_timestamp_seconds {int(time.time())}"] + with open(TEXTFILE + ".tmp", "w") as f: + f.write("\n".join(out) + "\n") + os.replace(TEXTFILE + ".tmp", TEXTFILE) + + +if METRICS: + run_metrics() + sys.exit(0) + days = [date.today() - timedelta(days=i) for i in range(7 if WEEKLY else 1, 0, -1)] stamps = [d.strftime("%d/%b/%Y") for d in days] diff --git a/ansible/roles/crawler_alert/files/crawler-metrics.service b/ansible/roles/crawler_alert/files/crawler-metrics.service new file mode 100644 index 0000000..aad2da2 --- /dev/null +++ b/ansible/roles/crawler_alert/files/crawler-metrics.service @@ -0,0 +1,12 @@ +[Unit] +Description=Crawler counters for Prometheus (node_exporter textfile) +After=docker.service + +[Service] +Type=oneshot +EnvironmentFile=/etc/crawler-alert.env +# Every router, not only Forgejo's; fewer DNS lookups per 5-minute run +Environment=CRAWLER_METRICS_ROUTER= CRAWLER_RDNS_MAX=100 +ExecStart=/usr/local/bin/crawler-alert.py --metrics +Nice=10 +TimeoutStartSec=4min diff --git a/ansible/roles/crawler_alert/files/crawler-metrics.timer b/ansible/roles/crawler_alert/files/crawler-metrics.timer new file mode 100644 index 0000000..2b20a3e --- /dev/null +++ b/ansible/roles/crawler_alert/files/crawler-metrics.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Crawler counters every 5 minutes + +[Timer] +OnCalendar=*:0/5 +RandomizedDelaySec=30 + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/crawler_alert/tasks/main.yaml b/ansible/roles/crawler_alert/tasks/main.yaml index 85cd559..ab33a22 100644 --- a/ansible/roles/crawler_alert/tasks/main.yaml +++ b/ansible/roles/crawler_alert/tasks/main.yaml @@ -22,6 +22,8 @@ - crawler-alert.timer - crawler-weekly.service - crawler-weekly.timer + - crawler-metrics.service + - crawler-metrics.timer notify: Reload systemd - name: Flush handlers @@ -35,3 +37,4 @@ loop: - crawler-alert.timer - crawler-weekly.timer + - crawler-metrics.timer diff --git a/ansible/roles/crawler_alert/templates/crawler-alert.env.j2 b/ansible/roles/crawler_alert/templates/crawler-alert.env.j2 index 7677840..a7186be 100644 --- a/ansible/roles/crawler_alert/templates/crawler-alert.env.j2 +++ b/ansible/roles/crawler_alert/templates/crawler-alert.env.j2 @@ -6,3 +6,5 @@ CRAWLER_THRESHOLD={{ crawler_threshold }} CRAWLER_MAIL_CONTAINER=forgejo CRAWLER_MAIL_INI=/data/gitea/conf/app.ini CRAWLER_ROBOTS={{ crawler_robots_url }} +CRAWLER_LIVE_LOG=/opt/traefik/logs/access.log +CRAWLER_TEXTFILE={{ monitoring_textfile_dir }}/crawler.prom diff --git a/ansible/roles/monitoring/files/dashboards/Services/crawlers.json b/ansible/roles/monitoring/files/dashboards/Services/crawlers.json new file mode 100644 index 0000000..88a1783 --- /dev/null +++ b/ansible/roles/monitoring/files/dashboards/Services/crawlers.json @@ -0,0 +1,1148 @@ +{ + "uid": "asxpio-crawlers", + "title": "AI crawlers", + "tags": [ + "asxpio", + "traefik" + ], + "timezone": "browser", + "editable": false, + "refresh": "5m", + "time": { + "from": "now-24h", + "to": "now" + }, + "schemaVersion": 39, + "graphTooltip": 1, + "links": [ + { + "title": "Dashboards", + "type": "dashboards", + "asDropdown": true, + "includeVars": false, + "keepTime": true, + "tags": [] + } + ], + "panels": [ + { + "type": "piechart", + "title": "Who is asking (time range)", + "id": 1, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(increase(crawler_ai_requests_total{verdict=\"verified\"}[$__range]))", + "legendFormat": "AI · verified", + "refId": "A", + "instant": true, + "range": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(increase(crawler_ai_requests_total{verdict=\"spoofed\"}[$__range]))", + "legendFormat": "AI · spoofed", + "refId": "B", + "instant": true, + "range": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(increase(crawler_ai_requests_total{verdict=~\"unknown|proxied|private\"}[$__range]))", + "legendFormat": "AI · unverifiable", + "refId": "C", + "instant": true, + "range": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(increase(crawler_class_requests_total{class=\"other-bot\"}[$__range]))", + "legendFormat": "other bots", + "refId": "D", + "instant": true, + "range": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(increase(crawler_class_requests_total{class=\"browser-like\"}[$__range]))", + "legendFormat": "browser-like", + "refId": "E", + "instant": true, + "range": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(increase(crawler_class_requests_total{class=\"none\"}[$__range]))", + "legendFormat": "no user agent", + "refId": "F", + "instant": true, + "range": false + } + ], + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "verified" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "spoofed" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "proxied" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unknown" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "yellow" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "private" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "purple" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "AI · verified" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "AI · spoofed" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "AI · unverifiable" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "yellow" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "other bots" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "browser-like" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "purple" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "no user agent" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "text" + } + } + ] + } + ] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "pieType": "donut", + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "tooltip": { + "mode": "single" + } + }, + "description": "From the Traefik access log, every router (crawler-alert.py --metrics, 5-minute timer). A request naming an AI bot counts as verified only if its source IP is in the vendor's published ranges, or passes forward-confirmed reverse DNS, or (Meta) sits in the vendor's own ASN. spoofed = the claim failed that check; unverifiable = via Cloudflare, no PTR, or over the DNS budget." + }, + { + "type": "timeseries", + "title": "AI crawler requests by verdict (per minute)", + "id": 2, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (verdict) (rate(crawler_ai_requests_total[$__rate_interval])) * 60", + "legendFormat": "{{verdict}}", + "refId": "A", + "instant": false, + "range": true + } + ], + "gridPos": { + "h": 8, + "w": 9, + "x": 6, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "short", + "custom": { + "stacking": { + "mode": "normal" + }, + "fillOpacity": 40, + "lineWidth": 1 + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "verified" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "spoofed" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "proxied" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unknown" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "yellow" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "private" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "purple" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "AI · verified" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "AI · spoofed" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "AI · unverifiable" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "yellow" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "other bots" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "browser-like" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "purple" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "no user agent" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "text" + } + } + ] + } + ] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "From the Traefik access log, every router (crawler-alert.py --metrics, 5-minute timer). A request naming an AI bot counts as verified only if its source IP is in the vendor's published ranges, or passes forward-confirmed reverse DNS, or (Meta) sits in the vendor's own ASN. spoofed = the claim failed that check; unverifiable = via Cloudflare, no PTR, or over the DNS budget." + }, + { + "type": "bargauge", + "title": "Verified AI crawlers by vendor (time range)", + "id": 3, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sort_desc(sum by (vendor) (increase(crawler_ai_requests_total{verdict=\"verified\"}[$__range])))", + "legendFormat": "{{vendor}}", + "refId": "A", + "instant": true, + "range": false + } + ], + "gridPos": { + "h": 8, + "w": 9, + "x": 15, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "unit": "short", + "decimals": 0, + "min": 0 + }, + "overrides": [] + }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "values": false + }, + "showUnfilled": true, + "valueMode": "color" + } + }, + { + "type": "stat", + "title": "Spoofed share of AI claims", + "id": 4, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(increase(crawler_ai_requests_total{verdict=\"spoofed\"}[$__range])) / sum(increase(crawler_ai_requests_total[$__range]))", + "legendFormat": "spoofed", + "refId": "A", + "instant": true, + "range": false + } + ], + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "decimals": 1, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.25 + }, + { + "color": "red", + "value": 0.5 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "value_and_name", + "justifyMode": "center", + "orientation": "auto" + } + }, + { + "type": "stat", + "title": "Served to verified AI crawlers", + "id": 5, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(increase(crawler_ai_verified_bytes_total[$__range]))", + "legendFormat": "bytes", + "refId": "A", + "instant": true, + "range": false + } + ], + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "color": { + "mode": "fixed", + "fixedColor": "green" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "value_and_name", + "justifyMode": "center", + "orientation": "auto" + } + }, + { + "type": "stat", + "title": "All requests (time range)", + "id": 6, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(increase(crawler_requests_total[$__range]))", + "legendFormat": "requests", + "refId": "A", + "instant": true, + "range": false + } + ], + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "text" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "value_and_name", + "justifyMode": "center", + "orientation": "auto" + } + }, + { + "type": "stat", + "title": "Collector last ran", + "id": 7, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "time() - crawler_metrics_last_run_timestamp_seconds", + "legendFormat": "ago", + "refId": "A", + "instant": true, + "range": false + } + ], + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "decimals": 0, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 900 + }, + { + "color": "red", + "value": 3600 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "value_and_name", + "justifyMode": "center", + "orientation": "auto" + } + }, + { + "type": "timeseries", + "title": "Verified AI requests by vendor (per minute)", + "id": 8, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (vendor) (rate(crawler_ai_requests_total{verdict=\"verified\"}[$__rate_interval])) * 60", + "legendFormat": "{{vendor}}", + "refId": "A", + "instant": false, + "range": true + } + ], + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 13 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "short", + "custom": { + "stacking": { + "mode": "normal" + }, + "fillOpacity": 40, + "lineWidth": 1 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "Client classes (per minute)", + "id": 9, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(crawler_ai_requests_total[$__rate_interval])) * 60", + "legendFormat": "AI-bot claims", + "refId": "A", + "instant": false, + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (class) (rate(crawler_class_requests_total[$__rate_interval])) * 60", + "legendFormat": "{{class}}", + "refId": "B", + "instant": false, + "range": true + } + ], + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 13 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "short", + "custom": { + "stacking": { + "mode": "normal" + }, + "fillOpacity": 40, + "lineWidth": 1 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "table", + "title": "AI bots by verdict (time range)", + "id": 10, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (vendor, bot, verdict) (increase(crawler_ai_requests_total[$__range]))", + "legendFormat": "", + "refId": "A", + "instant": true, + "range": false + } + ], + "gridPos": { + "h": 11, + "w": 14, + "x": 0, + "y": 22 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "verified" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "spoofed" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unknown" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "yellow" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "proxied" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + } + ] + }, + "options": { + "showHeader": true, + "sortBy": [ + { + "displayName": "verified", + "desc": true + } + ], + "cellHeight": "sm" + }, + "transformations": [ + { + "id": "labelsToFields", + "options": {} + }, + { + "id": "groupingToMatrix", + "options": { + "columnField": "verdict", + "rowField": "bot", + "valueField": "Value" + } + }, + { + "id": "organize", + "options": { + "renameByName": { + "bot\\verdict": "bot" + } + } + } + ] + }, + { + "type": "bargauge", + "title": "Bandwidth to verified AI crawlers (time range)", + "id": 11, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sort_desc(sum by (vendor) (increase(crawler_ai_verified_bytes_total[$__range])))", + "legendFormat": "{{vendor}}", + "refId": "A", + "instant": true, + "range": false + } + ], + "gridPos": { + "h": 11, + "w": 10, + "x": 14, + "y": 22 + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "color": { + "mode": "continuous-BlPu" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "values": false + }, + "showUnfilled": true + } + } + ] +} diff --git a/ansible/roles/monitoring/files/dashboards/overview.json b/ansible/roles/monitoring/files/dashboards/overview.json index ac403ac..29758ec 100644 --- a/ansible/roles/monitoring/files/dashboards/overview.json +++ b/ansible/roles/monitoring/files/dashboards/overview.json @@ -403,7 +403,7 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "node_load1 / count by (host) (node_cpu_seconds_total{mode=\"idle\"})", + "expr": "sum by (host) (node_load1) / count by (host) (node_cpu_seconds_total{mode=\"idle\"})", "legendFormat": "{{host}}", "refId": "A", "instant": true, @@ -769,7 +769,7 @@ "uid": "prometheus" }, "expr": "topk(12, container_memory_working_set_bytes{name!=\"\"})", - "legendFormat": "{{host}} \u00b7 {{name}}", + "legendFormat": "{{host}} · {{name}}", "refId": "A", "instant": true, "range": false @@ -826,7 +826,7 @@ "uid": "prometheus" }, "expr": "topk(8, sum by (host, name) (rate(container_cpu_usage_seconds_total{name!=\"\"}[$__rate_interval])))", - "legendFormat": "{{host}} \u00b7 {{name}}", + "legendFormat": "{{host}} · {{name}}", "refId": "A", "instant": false, "range": true @@ -1321,7 +1321,7 @@ "uid": "prometheus" }, "expr": "(time() - restic_backup_last_success_timestamp_seconds) / 3600", - "legendFormat": "restic \u00b7 {{host}} \u00b7 {{repo}}", + "legendFormat": "restic · {{host}} · {{repo}}", "refId": "A", "instant": true, "range": false @@ -1332,7 +1332,7 @@ "uid": "prometheus" }, "expr": "(time() - s3_replicate_last_success_timestamp_seconds) / 3600", - "legendFormat": "replica hel1 \u2192 fsn1", + "legendFormat": "replica hel1 → fsn1", "refId": "B", "instant": true, "range": false @@ -1488,6 +1488,579 @@ "enableLogDetails": true, "dedupStrategy": "signature" } + }, + { + "type": "row", + "title": "AI crawlers", + "id": 30, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 59 + }, + "panels": [] + }, + { + "type": "piechart", + "title": "Who is asking (time range)", + "id": 31, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(increase(crawler_ai_requests_total{verdict=\"verified\"}[$__range]))", + "legendFormat": "AI · verified", + "refId": "A", + "instant": true, + "range": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(increase(crawler_ai_requests_total{verdict=\"spoofed\"}[$__range]))", + "legendFormat": "AI · spoofed", + "refId": "B", + "instant": true, + "range": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(increase(crawler_ai_requests_total{verdict=~\"unknown|proxied|private\"}[$__range]))", + "legendFormat": "AI · unverifiable", + "refId": "C", + "instant": true, + "range": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(increase(crawler_class_requests_total{class=\"other-bot\"}[$__range]))", + "legendFormat": "other bots", + "refId": "D", + "instant": true, + "range": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(increase(crawler_class_requests_total{class=\"browser-like\"}[$__range]))", + "legendFormat": "browser-like", + "refId": "E", + "instant": true, + "range": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(increase(crawler_class_requests_total{class=\"none\"}[$__range]))", + "legendFormat": "no user agent", + "refId": "F", + "instant": true, + "range": false + } + ], + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 60 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "verified" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "spoofed" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "proxied" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unknown" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "yellow" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "private" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "purple" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "AI · verified" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "AI · spoofed" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "AI · unverifiable" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "yellow" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "other bots" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "browser-like" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "purple" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "no user agent" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "text" + } + } + ] + } + ] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "pieType": "donut", + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "tooltip": { + "mode": "single" + } + }, + "description": "From the Traefik access log, every router (crawler-alert.py --metrics, 5-minute timer). A request naming an AI bot counts as verified only if its source IP is in the vendor's published ranges, or passes forward-confirmed reverse DNS, or (Meta) sits in the vendor's own ASN. spoofed = the claim failed that check; unverifiable = via Cloudflare, no PTR, or over the DNS budget.", + "links": [ + { + "title": "Crawler details", + "url": "/d/asxpio-crawlers?${__url_time_range}" + } + ] + }, + { + "type": "timeseries", + "title": "AI crawler requests by verdict (per minute)", + "id": 32, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (verdict) (rate(crawler_ai_requests_total[$__rate_interval])) * 60", + "legendFormat": "{{verdict}}", + "refId": "A", + "instant": false, + "range": true + } + ], + "gridPos": { + "h": 8, + "w": 9, + "x": 6, + "y": 60 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "short", + "custom": { + "stacking": { + "mode": "normal" + }, + "fillOpacity": 40, + "lineWidth": 1 + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "verified" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "spoofed" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "proxied" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unknown" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "yellow" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "private" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "purple" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "AI · verified" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "AI · spoofed" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "AI · unverifiable" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "yellow" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "other bots" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "browser-like" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "purple" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "no user agent" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "text" + } + } + ] + } + ] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "From the Traefik access log, every router (crawler-alert.py --metrics, 5-minute timer). A request naming an AI bot counts as verified only if its source IP is in the vendor's published ranges, or passes forward-confirmed reverse DNS, or (Meta) sits in the vendor's own ASN. spoofed = the claim failed that check; unverifiable = via Cloudflare, no PTR, or over the DNS budget." + }, + { + "type": "bargauge", + "title": "Verified AI crawlers by vendor (time range)", + "id": 33, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sort_desc(sum by (vendor) (increase(crawler_ai_requests_total{verdict=\"verified\"}[$__range])))", + "legendFormat": "{{vendor}}", + "refId": "A", + "instant": true, + "range": false + } + ], + "gridPos": { + "h": 8, + "w": 9, + "x": 15, + "y": 60 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "unit": "short", + "decimals": 0, + "min": 0 + }, + "overrides": [] + }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "values": false + }, + "showUnfilled": true, + "valueMode": "color" + } } ], "graphTooltip": 1, @@ -1501,4 +2074,4 @@ "tags": [] } ] -} \ No newline at end of file +}