diff --git a/README.md b/README.md index a5df110..04250aa 100644 --- a/README.md +++ b/README.md @@ -55,10 +55,33 @@ range list belonging to the vendor its user-agent names. | Perplexity | `perplexity.ai/perplexitybot.json` | | Apple | `search.developer.apple.com/applebot.json` | -Meta, ByteDance, Amazon, Diffbot, Cohere and Common Crawl publish no -range file. Their traffic is reported but cannot be verified this way; -Amazon supports reverse-DNS verification instead, which this script -does not currently implement. +Meta, ByteDance, Amazon and You.com publish no range file. Those are +checked by **forward-confirmed reverse DNS** instead: the address's PTR +record must end in a vendor domain, *and* that hostname must resolve +back to the same address. The forward step is what makes it evidence — +a PTR alone is written by whoever controls the address block. + +| Bot | rDNS suffix | +|---|---| +| Amazonbot | `.crawl.amazonbot.amazon` | +| meta-externalagent, FacebookBot | `.facebook.com`, `.fbsv.net` | +| YouBot | `.search.you.com` | +| Bytespider | `.bytedance.com`, `.byteoversea.com` | +| PetalBot | `.petalsearch.com`, `.aspiegel.com` | +| DuckAssistBot | `.duckduckgo.com` | + +Diffbot, Cohere and Common Crawl publish neither ranges nor a +documented rDNS convention; they appear in the user-agent tally only. + +### Why there is no ASN verification + +ASN was considered and rejected. It looks like verification but is not: +on the sample instance, genuine YouBot (`68.67.112.227`) and genuine +Amazonbot (`100.24.167.60`) both resolve to **AS14618, Amazon** — the +same ASN as every EC2 instance on the internet, including any spoofer +renting one. An ASN match proves the traffic came from a cloud +provider, not from the vendor. Reporting that as "verified" would be +worse than reporting nothing. Ranges are fetched at run time and cached. If a vendor endpoint is unreachable the script falls back to the cache and says so in the mail, @@ -153,9 +176,19 @@ last updated in February 2025; Apple's dates to 2023. Genuine traffic from those two may fail verification. A 0% verified rate for a small vendor is weaker evidence than it looks. -**No reverse-DNS verification.** Amazonbot and Meta's crawlers can only -be checked by rDNS or ASN, neither of which is implemented. Their -traffic appears in the user-agent tally but not the verified table. +**Private source addresses cannot be judged at all.** If the logged +client IP is RFC1918 (`172.18.0.1`, a Docker bridge gateway, say), then +something in front of Traefik replaced the real address before it was +written. Those hits are counted in a `private` column, neither verified +nor spoofed. On the sample instance this was 21,624 of Meta's 21,651 +requests — the tool cannot tell you whether they were genuine, only +that the log does not contain the evidence. Fixing it is a Traefik +`forwardedHeaders` change, not a script change. + +**rDNS is rate-limited by design.** Each unique address costs two DNS +lookups, so only the busiest `CRAWLER_RDNS_MAX` (default 400) addresses +per bot are resolved; the rest are reported as unresolved. Raise it if +your DNS resolver is fast and local. **IPv6 is verified only where vendors publish v6 prefixes**, which most do; addresses outside those are treated as unverified. diff --git a/crawler-alert.env.example b/crawler-alert.env.example index 244d8f8..fc2cd81 100644 --- a/crawler-alert.env.example +++ b/crawler-alert.env.example @@ -38,3 +38,12 @@ CRAWLER_MAIL_INI=/data/gitea/conf/app.ini #CRAWLER_SMTP_FROM=alerts@example.com # smtp+starttls (default) | smtps | smtp (no TLS, local relay) #CRAWLER_SMTP_PROTOCOL=smtp+starttls + +# --- reverse DNS ------------------------------------------------------ +# FCrDNS verification for vendors that publish no IP ranges +# (Amazonbot, meta-externalagent, YouBot, Bytespider, PetalBot). +# Set to 0 to disable if your resolver is slow or unavailable. +CRAWLER_RDNS=1 +# Max unique addresses resolved per bot per run. Each costs two DNS +# lookups; the busiest addresses are resolved first. +CRAWLER_RDNS_MAX=400 diff --git a/crawler-alert.py b/crawler-alert.py index 2a0a359..5419825 100755 --- a/crawler-alert.py +++ b/crawler-alert.py @@ -13,7 +13,7 @@ Modes: Configuration is by environment variable; see README.md. The only value with no sensible default is CRAWLER_RECIPIENT. """ -import configparser, glob, gzip, ipaddress, json, os, re, smtplib, subprocess, sys, urllib.request +import configparser, glob, gzip, ipaddress, json, os, re, smtplib, socket, subprocess, sys, urllib.request from collections import Counter from datetime import date, timedelta from email.mime.text import MIMEText @@ -62,6 +62,31 @@ BOT_VENDOR = { "Applebot-Extended": "apple", "Applebot": "apple", "Googlebot": "google", "Google-Extended": "google", "GoogleOther": "google", } + +# --- reverse-DNS verification ------------------------------------------- +# For vendors that publish no IP ranges. The check is forward-confirmed +# (FCrDNS): PTR of the address must end in one of the vendor's domains, +# AND that hostname must resolve back to the same address. The forward +# step is what makes it meaningful - a PTR record alone is set by +# whoever controls the address block, so it proves nothing on its own. +BOT_RDNS = { + "Amazonbot": (".crawl.amazonbot.amazon",), + "meta-externalagent": (".facebook.com", ".fbsv.net"), + "meta-externalfetcher": (".facebook.com", ".fbsv.net"), + "FacebookBot": (".facebook.com", ".fbsv.net"), + "YouBot": (".search.you.com",), + "Bytespider": (".bytedance.com", ".byteoversea.com"), + "PetalBot": (".petalsearch.com", ".aspiegel.com"), + "DuckAssistBot": (".duckduckgo.com",), + # Apple and Google publish ranges, but also answer rDNS; listed so a + # range miss can still be resolved when the published file is stale. + "Applebot": (".applebot.apple.com",), + "Googlebot": (".googlebot.com", ".google.com"), +} +# rDNS is slow (two lookups per address), so only unique IPs are probed +# and results are memoised for the run. +RDNS_MAX = int(os.environ.get("CRAWLER_RDNS_MAX", "400")) +RDNS_ENABLE = os.environ.get("CRAWLER_RDNS", "1") != "0" CACHE = os.environ.get("CRAWLER_CACHE", "/var/cache/crawler-alert/ranges.json") CF_URL = "https://www.cloudflare.com/ips-v4" # claude.com and cloudflare.com sit behind Cloudflare, which 403s the @@ -129,6 +154,41 @@ def in_index(idx_v, addr): key = int(addr) >> 24 if addr.version == 4 else -1 return any(addr in n for n in idx_v.get(key, [])) + +_rdns_cache = {} + + +def fcrdns(ip, suffixes): + """Forward-confirmed reverse DNS. + + PTR must end in one of `suffixes`, and the name it gives must resolve + back to `ip`. Returns (ok, hostname). Without the forward step this + would only prove the address owner can write their own PTR record. + """ + key = (ip, suffixes) + if key in _rdns_cache: + return _rdns_cache[key] + result = (False, "") + try: + host = socket.gethostbyaddr(ip)[0].rstrip(".").lower() + if any(host.endswith(s) or host == s.lstrip(".") for s in suffixes): + _, _, addrs = socket.gethostbyname_ex(host) + try: + v6 = socket.getaddrinfo(host, None, socket.AF_INET6) + addrs = addrs + [a[4][0] for a in v6] + except OSError: + pass + if ip in addrs: + result = (True, host) + else: + result = (False, host + " (no forward match)") + else: + result = (False, host) + except OSError: + result = (False, "") + _rdns_cache[key] = result + return result + days = [date.today() - timedelta(days=i) for i in range(7 if WEEKLY else 1, 0, -1)] stamps = [d.strftime("%d/%b/%Y") for d in days] @@ -142,6 +202,9 @@ spoofed = Counter() # bot -> UA claimed, IP outside vendor ranges proxied = Counter() # bot -> arrived via Cloudflare, unverifiable ver_paths = Counter() # paths fetched by verified bots ver_ips = {} # bot -> set of verified source IPs +# rDNS-verified vendors: {bot: {ip: hits}}, resolved after the log pass +rdns_hits = {b: Counter() for b in BOT_RDNS} +rdns_proxied = Counter() for path in sorted(glob.glob(LOG_GLOB)): opener = gzip.open if path.endswith(".gz") else open with opener(path, "rt", errors="replace") as f: @@ -182,11 +245,58 @@ for path in sorted(glob.glob(LOG_GLOB)): ver_paths[parts[6][:60]] += 1 else: spoofed[bot] += 1 + if WEEKLY and RDNS_ENABLE: + rb = next((b for b in sorted(BOT_RDNS, key=len, reverse=True) + if b in u), None) + # skip if already counted by IP-range check + if rb and not (rb in BOT_VENDOR and rb == bot): + try: + a2 = ipaddress.ip_address(parts[0]) + except ValueError: + a2 = None + if a2 is None: + pass + elif any(a2 in n for n in CFNETS): + rdns_proxied[rb] += 1 + else: + rdns_hits[rb][parts[0]] += 1 break total = sum(v[0] for v in per_day.values()) all_ips = set().union(*(v[1] for v in per_day.values())) +# --- resolve rDNS candidates --------------------------------------------- +# One pass over unique addresses, busiest first, capped by RDNS_MAX so a +# flood of distinct forgeries cannot stall the report on DNS timeouts. +rdns_ok = Counter() # bot -> hits from forward-confirmed addresses +rdns_bad = Counter() # bot -> hits that failed confirmation +rdns_skipped = Counter() # bot -> hits left unresolved by the cap +rdns_private = Counter() # bot -> hits from RFC1918/loopback (see below) +rdns_ok_ips = {} +if WEEKLY and RDNS_ENABLE: + socket.setdefaulttimeout(3) + for bot, counter in rdns_hits.items(): + budget = RDNS_MAX + for ip, hits in counter.most_common(): + # A private source address means the real client IP never + # reached the log: something in front (Docker's bridge + # gateway, a local proxy) is rewriting it. Neither verified + # nor spoofed - the evidence simply is not there. + if ipaddress.ip_address(ip).is_private: + rdns_private[bot] += hits + continue + if budget <= 0: + rdns_skipped[bot] += hits + continue + budget -= 1 + ok, _host = fcrdns(ip, BOT_RDNS[bot]) + if ok: + rdns_ok[bot] += hits + rdns_ok_ips.setdefault(bot, set()).add(ip) + else: + rdns_bad[bot] += hits + socket.setdefaulttimeout(None) + if not WEEKLY: s0 = stamps[0] count, ips = per_day[s0][0], len(per_day[s0][1]) @@ -241,6 +351,33 @@ else: lines.append(" Top paths fetched by verified crawlers:") for p, c in ver_paths.most_common(8): lines.append(f" {c:>6} {p}") + + # --- vendors verified by reverse DNS --------------------------------- + if RDNS_ENABLE and (rdns_ok or rdns_bad or rdns_proxied): + lines.append("") + lines.append("-" * 62) + lines.append("VERIFIED BY REVERSE DNS (vendors publishing no IP ranges)") + lines.append("-" * 62) + lines.append(f" {'bot':<22}{'confirmed':>10}{'failed':>8}{'via CF':>8}" + f"{'private':>8} {'IPs':>4}") + for bot in sorted(set(rdns_ok) | set(rdns_bad) | set(rdns_proxied) | set(rdns_private), + key=lambda b: -(rdns_ok[b] + rdns_bad[b] + rdns_proxied[b] + + rdns_private[b])): + lines.append(f" {bot:<22}{rdns_ok[bot]:>10}{rdns_bad[bot]:>8}" + f"{rdns_proxied[bot]:>8}{rdns_private[bot]:>8}" + f" {len(rdns_ok_ips.get(bot, ())):>4}") + if rdns_skipped: + sk = ", ".join(f"{b} {c}" for b, c in rdns_skipped.most_common(4)) + lines.append(f" (unresolved, over CRAWLER_RDNS_MAX={RDNS_MAX}: {sk})") + lines.append("") + lines.append(" Forward-confirmed: the address's PTR record ends in a vendor") + lines.append(" domain AND that hostname resolves back to the same address.") + lines.append(" 'failed' means the PTR was absent, pointed elsewhere, or did") + lines.append(" not confirm - i.e. the user-agent is unsupported by DNS.") + if sum(rdns_private.values()): + lines.append(" 'private' means the logged source was an RFC1918 address:") + lines.append(" a proxy or Docker bridge is masking the real client, so") + lines.append(" these cannot be judged either way. See README.") body = "\n".join(lines) + "\n" subject = f"{SITE} weekly crawler report: {total} requests, {len(all_ips)} IPs" if DRY: