From dcacd8565523c7af4e76e700a454507c9ec4004a Mon Sep 17 00:00:00 2001 From: Sergei Poljanski Date: Tue, 11 Aug 2026 02:42:09 +0400 Subject: [PATCH] Add crawler statistics script with IP-verified bot reporting The Traefik access log shows a large volume of requests carrying AI crawler user-agents, but a user-agent header is self-declared and free to forge. Over 2.27M requests (Nov 2025 - Aug 2026), most of that traffic did not originate from the vendor it named: Claude-User was 4992/5000 forged, ChatGPT-User 2657/2670, Google-Extended 747/747. A pool of roughly 100 Google Cloud hosts rotates through six vendor identities and requests /.env, /terraform.tfstate and /serviceAccountKey.json - credential harvesting, not crawling. Counting these by user-agent alone makes the report worse than useless: it attributes scraper load to vendors that never sent it, and hides the fact that genuine crawler traffic is roughly 0.1% of requests and fully robots.txt-compliant. Verify instead against the vendors' published IP range lists (OpenAI, Anthropic, Google, Perplexity, Apple) and split each bot into verified / spoofed / via-Cloudflare buckets. Requests arriving through Cloudflare are reported separately rather than guessed at, since the logged address is the CF edge and not the client. Ranges are fetched at run time and cached, with fallback to the cache when a vendor endpoint is unreachable, so an outage degrades the report rather than breaking the weekly mail. The fetch sends an explicit User-Agent: claude.com and cloudflare.com sit behind Cloudflare, which 403s the default Python-urllib agent. Only --weekly performs the fetch; the daily threshold alert is unchanged and still runs without network access. Assisted-by: Claude:opus-5 --- crawler-alert.py | 243 +++++++++++++++++++++++++++++++++++++++++ crawler-alert.service | 7 ++ crawler-alert.timer | 9 ++ crawler-weekly.service | 7 ++ crawler-weekly.timer | 9 ++ 5 files changed, 275 insertions(+) create mode 100755 crawler-alert.py create mode 100644 crawler-alert.service create mode 100644 crawler-alert.timer create mode 100644 crawler-weekly.service create mode 100644 crawler-weekly.timer diff --git a/crawler-alert.py b/crawler-alert.py new file mode 100755 index 0000000..4abc9e8 --- /dev/null +++ b/crawler-alert.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +"""Forgejo crawler watch on git.asxp.io. Two modes: + (default) daily threshold alert: mail only if yesterday > 100k requests + --weekly Monday summary: always mail stats for the past 7 days + --dry-run print instead of mailing (combinable with --weekly) +Set up 2026-07-15; timers: crawler-alert.timer, crawler-weekly.timer.""" +import configparser, glob, gzip, ipaddress, json, os, re, smtplib, subprocess, sys, urllib.request +from collections import Counter +from datetime import date, timedelta +from email.mime.text import MIMEText + +THRESHOLD = 100_000 +RECIPIENT = "me@asxp.io" +WEEKLY = "--weekly" in sys.argv +DRY = "--dry-run" in sys.argv + +# --- verified-crawler support ------------------------------------------- +# Vendors publishing machine-readable IP ranges. A UA token alone proves +# nothing: as of Aug 2026 ~77% of AI-bot-labelled traffic here came from +# hosts outside these ranges (mostly GCP). Only IP-verified hits count. +RANGE_SOURCES = { + "openai": ["https://openai.com/chatgpt-user.json", + "https://openai.com/gptbot.json", + "https://openai.com/searchbot.json"], + "anthropic": ["https://claude.com/crawling/bots.json"], + "perplexity":["https://www.perplexity.ai/perplexitybot.json"], + "apple": ["https://search.developer.apple.com/applebot.json"], + "google": ["https://developers.google.com/static/crawling/ipranges/special-crawlers.json", + "https://developers.google.com/static/crawling/ipranges/common-crawlers.json", + "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers.json", + "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers-google.json"], +} +# UA token -> vendor whose ranges must contain the source IP +BOT_VENDOR = { + "ChatGPT-User": "openai", "GPTBot": "openai", "OAI-SearchBot": "openai", + "ClaudeBot": "anthropic", "Claude-User": "anthropic", "Claude-SearchBot": "anthropic", + "PerplexityBot": "perplexity", "Perplexity-User": "perplexity", + "Applebot-Extended": "apple", "Applebot": "apple", + "Googlebot": "google", "Google-Extended": "google", "GoogleOther": "google", +} +CACHE = "/var/cache/crawler-alert/ranges.json" +CF_URL = "https://www.cloudflare.com/ips-v4" +# claude.com and cloudflare.com sit behind Cloudflare, which 403s the +# default Python-urllib UA. Send a real one. +UA = "Mozilla/5.0 (compatible; crawler-alert/1.0; +https://git.asxp.io)" + + +def fetch(url, timeout=20): + req = urllib.request.Request(url, headers={"User-Agent": UA}) + with urllib.request.urlopen(req, timeout=timeout) as r: + return r.read() + + +def load_ranges(): + """Fetch vendor prefixes; fall back to cache so a vendor outage + degrades the report instead of breaking the mail.""" + out, stale = {}, [] + for vendor, urls in RANGE_SOURCES.items(): + nets = [] + for u in urls: + try: + for p in json.loads(fetch(u)).get("prefixes", []): + for k, v in p.items(): + if k.startswith("ipv"): + nets.append(v) + except Exception: + stale.append(vendor) + if nets: + out[vendor] = nets + cf = [] + try: + cf = [l.strip() for l in fetch(CF_URL).decode().splitlines() if l.strip()] + except Exception: + stale.append("cloudflare") + try: + cached = json.load(open(CACHE)) + except Exception: + cached = {} + for vendor in RANGE_SOURCES: + if vendor not in out and vendor in cached.get("vendors", {}): + out[vendor] = cached["vendors"][vendor] + if not cf: + cf = cached.get("cloudflare", []) + if out: + try: + os.makedirs(os.path.dirname(CACHE), exist_ok=True) + json.dump({"vendors": out, "cloudflare": cf}, open(CACHE, "w")) + except Exception: + pass + # bucket by first octet so lookups stay cheap over ~1425 prefixes + idx = {} + for vendor, nets in out.items(): + b = {} + for s in nets: + n = ipaddress.ip_network(s) + b.setdefault(int(n.network_address) >> 24 if n.version == 4 else -1, []).append(n) + idx[vendor] = b + cfnets = [ipaddress.ip_network(s) for s in cf] + return idx, cfnets, sorted(set(stale)) + + +def in_index(idx_v, addr): + if idx_v is None: + return False + key = int(addr) >> 24 if addr.version == 4 else -1 + return any(addr in n for n in idx_v.get(key, [])) + +days = [date.today() - timedelta(days=i) for i in range(7 if WEEKLY else 1, 0, -1)] +stamps = [d.strftime("%d/%b/%Y") for d in days] + +RANGES, CFNETS, STALE = (load_ranges() if WEEKLY else ({}, [], [])) + +per_day = {s: [0, set()] for s in stamps} +repos = Counter() +uas = Counter() +verified = Counter() # bot -> IP-verified hits +spoofed = Counter() # bot -> UA claimed, IP outside vendor ranges +proxied = Counter() # bot -> arrived via Cloudflare, unverifiable +ver_paths = Counter() # paths fetched by verified bots +ver_ips = {} # bot -> set of verified source IPs +for path in sorted(glob.glob("/opt/traefik/logs/access.log*")): + opener = gzip.open if path.endswith(".gz") else open + with opener(path, "rt", errors="replace") as f: + for line in f: + if "forgejo@docker" not in line: + continue + for s in stamps: + if s in line: + per_day[s][0] += 1 + per_day[s][1].add(line.split(" ", 1)[0]) + parts = line.split() + if len(parts) > 6 and parts[6].count("/") >= 2: + repos["/".join(parts[6].split("/")[1:3])] += 1 + ua = re.search(r"\" \"([^\"]*)\" \d+ \"forgejo", line) + if ua: + u = ua.group(1) + m = re.search(r"(bot|crawler|spider|scrapy|externalagent|gpt|claude|perplexity)[\w./-]*", u, re.I) + uas[m.group(0) if m else ("(none)" if u == "-" else "(browser-like)")] += 1 + if WEEKLY: + # longest token first so Applebot-Extended wins over Applebot + bot = next((b for b in sorted(BOT_VENDOR, key=len, reverse=True) if b in u), None) + if bot: + try: + addr = ipaddress.ip_address(parts[0]) + except ValueError: + addr = None + if addr is None: + pass + elif any(addr in n for n in CFNETS): + proxied[bot] += 1 + elif in_index(RANGES.get(BOT_VENDOR[bot]), addr): + verified[bot] += 1 + ver_ips.setdefault(bot, set()).add(parts[0]) + if len(parts) > 6: + ver_paths[parts[6][:60]] += 1 + else: + spoofed[bot] += 1 + break + +total = sum(v[0] for v in per_day.values()) +all_ips = set().union(*(v[1] for v in per_day.values())) + +if not WEEKLY: + s0 = stamps[0] + count, ips = per_day[s0][0], len(per_day[s0][1]) + print(f"{s0}: {count} forgejo requests, {ips} unique IPs (threshold {THRESHOLD})") + if DRY or count <= THRESHOLD: + sys.exit(0) + subject = f"git.asxp.io: {count} requests on {s0} (>100k/day threshold)" + body = (f"Forgejo served {count} requests from {ips} unique IPs on {s0} - " + f"above the {THRESHOLD}/day watch threshold.\n\n" + "Time to consider Anubis (PoW challenge) in front of Forgejo.\n") +else: + lines = [f"git.asxp.io weekly crawler report ({stamps[0]} - {stamps[-1]})", ""] + lines.append(f"Total: {total} requests, {len(all_ips)} unique IPs") + lines.append(f"Daily threshold alert fires above {THRESHOLD} req/day (none = quiet week)") + lines.append("") + lines.append("Per day:") + for s in stamps: + lines.append(f" {s}: {per_day[s][0]:>7} requests, {len(per_day[s][1]):>6} unique IPs") + lines.append("") + lines.append("Top repos:") + for r, c in repos.most_common(5): + lines.append(f" {c:>7} {r}") + lines.append("") + lines.append("User-agent classes (browser-like = mostly the spoofing swarm):") + for u, c in uas.most_common(8): + lines.append(f" {c:>7} {u}") + lines.append("") + + # --- legitimate (IP-verified) crawlers ------------------------------- + lines.append("=" * 62) + lines.append("LEGITIMATE CRAWLERS (source IP inside vendor's published ranges)") + lines.append("=" * 62) + if STALE: + lines.append(f"NOTE: range fetch failed for {', '.join(STALE)} - used cache") + tv, ts, tp = sum(verified.values()), sum(spoofed.values()), sum(proxied.values()) + claimed = tv + ts + tp + if not claimed: + lines.append(" No AI-bot user-agents seen this week.") + else: + lines.append(f" {'bot':<20}{'verified':>9}{'spoofed':>9}{'via CF':>8} {'IPs':>4}") + for bot in sorted(set(verified) | set(spoofed) | set(proxied), + key=lambda b: -(verified[b] + spoofed[b] + proxied[b])): + lines.append(f" {bot:<20}{verified[bot]:>9}{spoofed[bot]:>9}" + f"{proxied[bot]:>8} {len(ver_ips.get(bot, ())):>4}") + pct = tv / claimed * 100 + lines.append("") + lines.append(f" {claimed} requests claimed an AI-bot identity; {tv} verified ({pct:.1f}%).") + lines.append(f" {ts} failed IP verification (spoofed). {tp} arrived via Cloudflare") + lines.append(" and cannot be verified by IP - not counted either way.") + if ver_paths: + lines.append("") + lines.append(" Top paths fetched by verified crawlers:") + for p, c in ver_paths.most_common(8): + lines.append(f" {c:>6} {p}") + body = "\n".join(lines) + "\n" + subject = f"git.asxp.io weekly crawler report: {total} requests, {len(all_ips)} IPs" + if DRY: + print(subject); print(); print(body) + sys.exit(0) + +ini = subprocess.run(["docker","exec","forgejo","cat","/data/gitea/conf/app.ini"], + capture_output=True, text=True).stdout +cp = configparser.ConfigParser(interpolation=None, strict=False) +cp.read_string("[DEFAULT]\n" + ini) +m = cp["mailer"] +proto = m.get("PROTOCOL", "smtps").strip() +addr, port = m.get("SMTP_ADDR").strip(), int(m.get("SMTP_PORT", "465").strip()) +user, passwd = m.get("USER").strip(), m.get("PASSWD").strip().strip("`\"") +sender = m.get("FROM", user).strip() + +msg = MIMEText(body) +msg["Subject"], msg["From"], msg["To"] = subject, sender, RECIPIENT +if proto == "smtps": + s = smtplib.SMTP_SSL(addr, port, timeout=30) +else: + s = smtplib.SMTP(addr, port, timeout=30) + s.starttls() +s.login(user, passwd) +s.sendmail(sender, [RECIPIENT], msg.as_string()) +s.quit() +print(f"mailed to {RECIPIENT}") diff --git a/crawler-alert.service b/crawler-alert.service new file mode 100644 index 0000000..9407edb --- /dev/null +++ b/crawler-alert.service @@ -0,0 +1,7 @@ +[Unit] +Description=Mail alert when Forgejo exceeds 100k requests/day +After=docker.service + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/crawler-alert.py diff --git a/crawler-alert.timer b/crawler-alert.timer new file mode 100644 index 0000000..0b8b4f5 --- /dev/null +++ b/crawler-alert.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Daily Forgejo crawler volume check + +[Timer] +OnCalendar=*-*-* 00:15:00 +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/crawler-weekly.service b/crawler-weekly.service new file mode 100644 index 0000000..93349bd --- /dev/null +++ b/crawler-weekly.service @@ -0,0 +1,7 @@ +[Unit] +Description=Weekly Forgejo crawler report mail +After=docker.service + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/crawler-alert.py --weekly diff --git a/crawler-weekly.timer b/crawler-weekly.timer new file mode 100644 index 0000000..2982024 --- /dev/null +++ b/crawler-weekly.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Weekly Forgejo crawler report, Mondays 12:00 UTC + +[Timer] +OnCalendar=Mon *-*-* 12:00:00 +Persistent=true + +[Install] +WantedBy=timers.target