Add crawler statistics script with IP-verified bot reporting
The Traefik access log shows a large volume of requests carrying AI crawler user-agents, but a user-agent header is self-declared and free to forge. Over 2.27M requests (Nov 2025 - Aug 2026), most of that traffic did not originate from the vendor it named: Claude-User was 4992/5000 forged, ChatGPT-User 2657/2670, Google-Extended 747/747. A pool of roughly 100 Google Cloud hosts rotates through six vendor identities and requests /.env, /terraform.tfstate and /serviceAccountKey.json - credential harvesting, not crawling. Counting these by user-agent alone makes the report worse than useless: it attributes scraper load to vendors that never sent it, and hides the fact that genuine crawler traffic is roughly 0.1% of requests and fully robots.txt-compliant. Verify instead against the vendors' published IP range lists (OpenAI, Anthropic, Google, Perplexity, Apple) and split each bot into verified / spoofed / via-Cloudflare buckets. Requests arriving through Cloudflare are reported separately rather than guessed at, since the logged address is the CF edge and not the client. Ranges are fetched at run time and cached, with fallback to the cache when a vendor endpoint is unreachable, so an outage degrades the report rather than breaking the weekly mail. The fetch sends an explicit User-Agent: claude.com and cloudflare.com sit behind Cloudflare, which 403s the default Python-urllib agent. Only --weekly performs the fetch; the daily threshold alert is unchanged and still runs without network access. Assisted-by: Claude:opus-5
This commit is contained in:
parent
fc3cb81500
commit
dcacd85655
GPG key ID:
4F8851660FA4121B
5 changed files with 275 additions and 0 deletions
243
crawler-alert.py
Executable file
243
crawler-alert.py
Executable file
|
|
@ -0,0 +1,243 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Forgejo crawler watch on git.asxp.io. Two modes:
|
||||||
|
(default) daily threshold alert: mail only if yesterday > 100k requests
|
||||||
|
--weekly Monday summary: always mail stats for the past 7 days
|
||||||
|
--dry-run print instead of mailing (combinable with --weekly)
|
||||||
|
Set up 2026-07-15; timers: crawler-alert.timer, crawler-weekly.timer."""
|
||||||
|
import configparser, glob, gzip, ipaddress, json, os, re, smtplib, subprocess, sys, urllib.request
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import date, timedelta
|
||||||
|
from email.mime.text import MIMEText
|
||||||
|
|
||||||
|
THRESHOLD = 100_000
|
||||||
|
RECIPIENT = "me@asxp.io"
|
||||||
|
WEEKLY = "--weekly" in sys.argv
|
||||||
|
DRY = "--dry-run" in sys.argv
|
||||||
|
|
||||||
|
# --- verified-crawler support -------------------------------------------
|
||||||
|
# Vendors publishing machine-readable IP ranges. A UA token alone proves
|
||||||
|
# nothing: as of Aug 2026 ~77% of AI-bot-labelled traffic here came from
|
||||||
|
# hosts outside these ranges (mostly GCP). Only IP-verified hits count.
|
||||||
|
RANGE_SOURCES = {
|
||||||
|
"openai": ["https://openai.com/chatgpt-user.json",
|
||||||
|
"https://openai.com/gptbot.json",
|
||||||
|
"https://openai.com/searchbot.json"],
|
||||||
|
"anthropic": ["https://claude.com/crawling/bots.json"],
|
||||||
|
"perplexity":["https://www.perplexity.ai/perplexitybot.json"],
|
||||||
|
"apple": ["https://search.developer.apple.com/applebot.json"],
|
||||||
|
"google": ["https://developers.google.com/static/crawling/ipranges/special-crawlers.json",
|
||||||
|
"https://developers.google.com/static/crawling/ipranges/common-crawlers.json",
|
||||||
|
"https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers.json",
|
||||||
|
"https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers-google.json"],
|
||||||
|
}
|
||||||
|
# UA token -> vendor whose ranges must contain the source IP
|
||||||
|
BOT_VENDOR = {
|
||||||
|
"ChatGPT-User": "openai", "GPTBot": "openai", "OAI-SearchBot": "openai",
|
||||||
|
"ClaudeBot": "anthropic", "Claude-User": "anthropic", "Claude-SearchBot": "anthropic",
|
||||||
|
"PerplexityBot": "perplexity", "Perplexity-User": "perplexity",
|
||||||
|
"Applebot-Extended": "apple", "Applebot": "apple",
|
||||||
|
"Googlebot": "google", "Google-Extended": "google", "GoogleOther": "google",
|
||||||
|
}
|
||||||
|
CACHE = "/var/cache/crawler-alert/ranges.json"
|
||||||
|
CF_URL = "https://www.cloudflare.com/ips-v4"
|
||||||
|
# claude.com and cloudflare.com sit behind Cloudflare, which 403s the
|
||||||
|
# default Python-urllib UA. Send a real one.
|
||||||
|
UA = "Mozilla/5.0 (compatible; crawler-alert/1.0; +https://git.asxp.io)"
|
||||||
|
|
||||||
|
|
||||||
|
def fetch(url, timeout=20):
|
||||||
|
req = urllib.request.Request(url, headers={"User-Agent": UA})
|
||||||
|
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||||
|
return r.read()
|
||||||
|
|
||||||
|
|
||||||
|
def load_ranges():
|
||||||
|
"""Fetch vendor prefixes; fall back to cache so a vendor outage
|
||||||
|
degrades the report instead of breaking the mail."""
|
||||||
|
out, stale = {}, []
|
||||||
|
for vendor, urls in RANGE_SOURCES.items():
|
||||||
|
nets = []
|
||||||
|
for u in urls:
|
||||||
|
try:
|
||||||
|
for p in json.loads(fetch(u)).get("prefixes", []):
|
||||||
|
for k, v in p.items():
|
||||||
|
if k.startswith("ipv"):
|
||||||
|
nets.append(v)
|
||||||
|
except Exception:
|
||||||
|
stale.append(vendor)
|
||||||
|
if nets:
|
||||||
|
out[vendor] = nets
|
||||||
|
cf = []
|
||||||
|
try:
|
||||||
|
cf = [l.strip() for l in fetch(CF_URL).decode().splitlines() if l.strip()]
|
||||||
|
except Exception:
|
||||||
|
stale.append("cloudflare")
|
||||||
|
try:
|
||||||
|
cached = json.load(open(CACHE))
|
||||||
|
except Exception:
|
||||||
|
cached = {}
|
||||||
|
for vendor in RANGE_SOURCES:
|
||||||
|
if vendor not in out and vendor in cached.get("vendors", {}):
|
||||||
|
out[vendor] = cached["vendors"][vendor]
|
||||||
|
if not cf:
|
||||||
|
cf = cached.get("cloudflare", [])
|
||||||
|
if out:
|
||||||
|
try:
|
||||||
|
os.makedirs(os.path.dirname(CACHE), exist_ok=True)
|
||||||
|
json.dump({"vendors": out, "cloudflare": cf}, open(CACHE, "w"))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
# bucket by first octet so lookups stay cheap over ~1425 prefixes
|
||||||
|
idx = {}
|
||||||
|
for vendor, nets in out.items():
|
||||||
|
b = {}
|
||||||
|
for s in nets:
|
||||||
|
n = ipaddress.ip_network(s)
|
||||||
|
b.setdefault(int(n.network_address) >> 24 if n.version == 4 else -1, []).append(n)
|
||||||
|
idx[vendor] = b
|
||||||
|
cfnets = [ipaddress.ip_network(s) for s in cf]
|
||||||
|
return idx, cfnets, sorted(set(stale))
|
||||||
|
|
||||||
|
|
||||||
|
def in_index(idx_v, addr):
|
||||||
|
if idx_v is None:
|
||||||
|
return False
|
||||||
|
key = int(addr) >> 24 if addr.version == 4 else -1
|
||||||
|
return any(addr in n for n in idx_v.get(key, []))
|
||||||
|
|
||||||
|
days = [date.today() - timedelta(days=i) for i in range(7 if WEEKLY else 1, 0, -1)]
|
||||||
|
stamps = [d.strftime("%d/%b/%Y") for d in days]
|
||||||
|
|
||||||
|
RANGES, CFNETS, STALE = (load_ranges() if WEEKLY else ({}, [], []))
|
||||||
|
|
||||||
|
per_day = {s: [0, set()] for s in stamps}
|
||||||
|
repos = Counter()
|
||||||
|
uas = Counter()
|
||||||
|
verified = Counter() # bot -> IP-verified hits
|
||||||
|
spoofed = Counter() # bot -> UA claimed, IP outside vendor ranges
|
||||||
|
proxied = Counter() # bot -> arrived via Cloudflare, unverifiable
|
||||||
|
ver_paths = Counter() # paths fetched by verified bots
|
||||||
|
ver_ips = {} # bot -> set of verified source IPs
|
||||||
|
for path in sorted(glob.glob("/opt/traefik/logs/access.log*")):
|
||||||
|
opener = gzip.open if path.endswith(".gz") else open
|
||||||
|
with opener(path, "rt", errors="replace") as f:
|
||||||
|
for line in f:
|
||||||
|
if "forgejo@docker" not in line:
|
||||||
|
continue
|
||||||
|
for s in stamps:
|
||||||
|
if s in line:
|
||||||
|
per_day[s][0] += 1
|
||||||
|
per_day[s][1].add(line.split(" ", 1)[0])
|
||||||
|
parts = line.split()
|
||||||
|
if len(parts) > 6 and parts[6].count("/") >= 2:
|
||||||
|
repos["/".join(parts[6].split("/")[1:3])] += 1
|
||||||
|
ua = re.search(r"\" \"([^\"]*)\" \d+ \"forgejo", line)
|
||||||
|
if ua:
|
||||||
|
u = ua.group(1)
|
||||||
|
m = re.search(r"(bot|crawler|spider|scrapy|externalagent|gpt|claude|perplexity)[\w./-]*", u, re.I)
|
||||||
|
uas[m.group(0) if m else ("(none)" if u == "-" else "(browser-like)")] += 1
|
||||||
|
if WEEKLY:
|
||||||
|
# longest token first so Applebot-Extended wins over Applebot
|
||||||
|
bot = next((b for b in sorted(BOT_VENDOR, key=len, reverse=True) if b in u), None)
|
||||||
|
if bot:
|
||||||
|
try:
|
||||||
|
addr = ipaddress.ip_address(parts[0])
|
||||||
|
except ValueError:
|
||||||
|
addr = None
|
||||||
|
if addr is None:
|
||||||
|
pass
|
||||||
|
elif any(addr in n for n in CFNETS):
|
||||||
|
proxied[bot] += 1
|
||||||
|
elif in_index(RANGES.get(BOT_VENDOR[bot]), addr):
|
||||||
|
verified[bot] += 1
|
||||||
|
ver_ips.setdefault(bot, set()).add(parts[0])
|
||||||
|
if len(parts) > 6:
|
||||||
|
ver_paths[parts[6][:60]] += 1
|
||||||
|
else:
|
||||||
|
spoofed[bot] += 1
|
||||||
|
break
|
||||||
|
|
||||||
|
total = sum(v[0] for v in per_day.values())
|
||||||
|
all_ips = set().union(*(v[1] for v in per_day.values()))
|
||||||
|
|
||||||
|
if not WEEKLY:
|
||||||
|
s0 = stamps[0]
|
||||||
|
count, ips = per_day[s0][0], len(per_day[s0][1])
|
||||||
|
print(f"{s0}: {count} forgejo requests, {ips} unique IPs (threshold {THRESHOLD})")
|
||||||
|
if DRY or count <= THRESHOLD:
|
||||||
|
sys.exit(0)
|
||||||
|
subject = f"git.asxp.io: {count} requests on {s0} (>100k/day threshold)"
|
||||||
|
body = (f"Forgejo served {count} requests from {ips} unique IPs on {s0} - "
|
||||||
|
f"above the {THRESHOLD}/day watch threshold.\n\n"
|
||||||
|
"Time to consider Anubis (PoW challenge) in front of Forgejo.\n")
|
||||||
|
else:
|
||||||
|
lines = [f"git.asxp.io weekly crawler report ({stamps[0]} - {stamps[-1]})", ""]
|
||||||
|
lines.append(f"Total: {total} requests, {len(all_ips)} unique IPs")
|
||||||
|
lines.append(f"Daily threshold alert fires above {THRESHOLD} req/day (none = quiet week)")
|
||||||
|
lines.append("")
|
||||||
|
lines.append("Per day:")
|
||||||
|
for s in stamps:
|
||||||
|
lines.append(f" {s}: {per_day[s][0]:>7} requests, {len(per_day[s][1]):>6} unique IPs")
|
||||||
|
lines.append("")
|
||||||
|
lines.append("Top repos:")
|
||||||
|
for r, c in repos.most_common(5):
|
||||||
|
lines.append(f" {c:>7} {r}")
|
||||||
|
lines.append("")
|
||||||
|
lines.append("User-agent classes (browser-like = mostly the spoofing swarm):")
|
||||||
|
for u, c in uas.most_common(8):
|
||||||
|
lines.append(f" {c:>7} {u}")
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
# --- legitimate (IP-verified) crawlers -------------------------------
|
||||||
|
lines.append("=" * 62)
|
||||||
|
lines.append("LEGITIMATE CRAWLERS (source IP inside vendor's published ranges)")
|
||||||
|
lines.append("=" * 62)
|
||||||
|
if STALE:
|
||||||
|
lines.append(f"NOTE: range fetch failed for {', '.join(STALE)} - used cache")
|
||||||
|
tv, ts, tp = sum(verified.values()), sum(spoofed.values()), sum(proxied.values())
|
||||||
|
claimed = tv + ts + tp
|
||||||
|
if not claimed:
|
||||||
|
lines.append(" No AI-bot user-agents seen this week.")
|
||||||
|
else:
|
||||||
|
lines.append(f" {'bot':<20}{'verified':>9}{'spoofed':>9}{'via CF':>8} {'IPs':>4}")
|
||||||
|
for bot in sorted(set(verified) | set(spoofed) | set(proxied),
|
||||||
|
key=lambda b: -(verified[b] + spoofed[b] + proxied[b])):
|
||||||
|
lines.append(f" {bot:<20}{verified[bot]:>9}{spoofed[bot]:>9}"
|
||||||
|
f"{proxied[bot]:>8} {len(ver_ips.get(bot, ())):>4}")
|
||||||
|
pct = tv / claimed * 100
|
||||||
|
lines.append("")
|
||||||
|
lines.append(f" {claimed} requests claimed an AI-bot identity; {tv} verified ({pct:.1f}%).")
|
||||||
|
lines.append(f" {ts} failed IP verification (spoofed). {tp} arrived via Cloudflare")
|
||||||
|
lines.append(" and cannot be verified by IP - not counted either way.")
|
||||||
|
if ver_paths:
|
||||||
|
lines.append("")
|
||||||
|
lines.append(" Top paths fetched by verified crawlers:")
|
||||||
|
for p, c in ver_paths.most_common(8):
|
||||||
|
lines.append(f" {c:>6} {p}")
|
||||||
|
body = "\n".join(lines) + "\n"
|
||||||
|
subject = f"git.asxp.io weekly crawler report: {total} requests, {len(all_ips)} IPs"
|
||||||
|
if DRY:
|
||||||
|
print(subject); print(); print(body)
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
ini = subprocess.run(["docker","exec","forgejo","cat","/data/gitea/conf/app.ini"],
|
||||||
|
capture_output=True, text=True).stdout
|
||||||
|
cp = configparser.ConfigParser(interpolation=None, strict=False)
|
||||||
|
cp.read_string("[DEFAULT]\n" + ini)
|
||||||
|
m = cp["mailer"]
|
||||||
|
proto = m.get("PROTOCOL", "smtps").strip()
|
||||||
|
addr, port = m.get("SMTP_ADDR").strip(), int(m.get("SMTP_PORT", "465").strip())
|
||||||
|
user, passwd = m.get("USER").strip(), m.get("PASSWD").strip().strip("`\"")
|
||||||
|
sender = m.get("FROM", user).strip()
|
||||||
|
|
||||||
|
msg = MIMEText(body)
|
||||||
|
msg["Subject"], msg["From"], msg["To"] = subject, sender, RECIPIENT
|
||||||
|
if proto == "smtps":
|
||||||
|
s = smtplib.SMTP_SSL(addr, port, timeout=30)
|
||||||
|
else:
|
||||||
|
s = smtplib.SMTP(addr, port, timeout=30)
|
||||||
|
s.starttls()
|
||||||
|
s.login(user, passwd)
|
||||||
|
s.sendmail(sender, [RECIPIENT], msg.as_string())
|
||||||
|
s.quit()
|
||||||
|
print(f"mailed to {RECIPIENT}")
|
||||||
7
crawler-alert.service
Normal file
7
crawler-alert.service
Normal file
|
|
@ -0,0 +1,7 @@
|
||||||
|
[Unit]
|
||||||
|
Description=Mail alert when Forgejo exceeds 100k requests/day
|
||||||
|
After=docker.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
ExecStart=/usr/local/bin/crawler-alert.py
|
||||||
9
crawler-alert.timer
Normal file
9
crawler-alert.timer
Normal file
|
|
@ -0,0 +1,9 @@
|
||||||
|
[Unit]
|
||||||
|
Description=Daily Forgejo crawler volume check
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnCalendar=*-*-* 00:15:00
|
||||||
|
Persistent=true
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
7
crawler-weekly.service
Normal file
7
crawler-weekly.service
Normal file
|
|
@ -0,0 +1,7 @@
|
||||||
|
[Unit]
|
||||||
|
Description=Weekly Forgejo crawler report mail
|
||||||
|
After=docker.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
ExecStart=/usr/local/bin/crawler-alert.py --weekly
|
||||||
9
crawler-weekly.timer
Normal file
9
crawler-weekly.timer
Normal file
|
|
@ -0,0 +1,9 @@
|
||||||
|
[Unit]
|
||||||
|
Description=Weekly Forgejo crawler report, Mondays 12:00 UTC
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnCalendar=Mon *-*-* 12:00:00
|
||||||
|
Persistent=true
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
Loading…
Add table
Add a link
Reference in a new issue