diff --git a/crawler-alert.env.example b/crawler-alert.env.example index fc2cd81..d17bd24 100644 --- a/crawler-alert.env.example +++ b/crawler-alert.env.example @@ -47,3 +47,9 @@ CRAWLER_RDNS=1 # Max unique addresses resolved per bot per run. Each costs two DNS # lookups; the busiest addresses are resolved first. CRAWLER_RDNS_MAX=400 + +# ASN fallback for vendors with no PTR records and no published ranges +# (currently Meta / AS32934 only). Used ONLY where the ASN is +# single-tenant; never for cloud ASNs where anyone can rent space. +# Requires `dig`. Set to 0 to disable. +CRAWLER_ASN=1 diff --git a/crawler-alert.py b/crawler-alert.py index 5419825..d3557f8 100755 --- a/crawler-alert.py +++ b/crawler-alert.py @@ -87,6 +87,25 @@ BOT_RDNS = { # and results are memoised for the run. RDNS_MAX = int(os.environ.get("CRAWLER_RDNS_MAX", "400")) RDNS_ENABLE = os.environ.get("CRAWLER_RDNS", "1") != "0" + +# --- ASN verification, for single-tenant networks only ------------------- +# An ASN match usually proves nothing: genuine YouBot and genuine +# Amazonbot both live in AS14618, which is also every EC2 instance a +# spoofer could rent. It is only evidence when the ASN belongs to the +# vendor exclusively and sells no compute in it. Meta qualifies - +# AS32934 is Meta's own network and cannot be rented - and Meta +# publishes no IP range file (their docs say to email webmasters@meta.com), +# so without this there is nothing to check them against at all. +BOT_ASN = { + "meta-externalagent": {"32934"}, + "meta-externalfetcher": {"32934"}, + "FacebookBot": {"32934"}, + "facebookexternalhit": {"32934"}, +} +ASN_ENABLE = os.environ.get("CRAWLER_ASN", "1") != "0" +# Team Cymru's DNS interface: .origin[6].asn.cymru.com TXT +CYMRU4 = "origin.asn.cymru.com" +CYMRU6 = "origin6.asn.cymru.com" CACHE = os.environ.get("CRAWLER_CACHE", "/var/cache/crawler-alert/ranges.json") CF_URL = "https://www.cloudflare.com/ips-v4" # claude.com and cloudflare.com sit behind Cloudflare, which 403s the @@ -155,6 +174,37 @@ def in_index(idx_v, addr): return any(addr in n for n in idx_v.get(key, [])) +_asn_cache = {} + + +def asn_of(ip): + """Origin ASN for an address via Team Cymru's DNS interface. + + Returns the ASN as a string, or "" if it cannot be determined. + Needs `dig`; absence degrades to no ASN evidence, never an error. + """ + if ip in _asn_cache: + return _asn_cache[ip] + asn = "" + try: + a = ipaddress.ip_address(ip) + if a.version == 4: + q = ".".join(reversed(ip.split("."))) + "." + CYMRU4 + else: + nibbles = a.exploded.replace(":", "") + q = ".".join(reversed(nibbles)) + "." + CYMRU6 + out = subprocess.run(["dig", "+short", "+time=3", "+tries=1", q, "TXT"], + capture_output=True, text=True, timeout=12).stdout + # "32934 | 2a03:2880:f814::/48 | IE | ripencc | 2011-08-22" + first = out.strip().strip('"').split("|")[0].strip() + if first.split(" ")[0].isdigit(): + asn = first.split(" ")[0] + except (OSError, ValueError, subprocess.SubprocessError): + pass + _asn_cache[ip] = asn + return asn + + _rdns_cache = {} @@ -168,7 +218,7 @@ def fcrdns(ip, suffixes): key = (ip, suffixes) if key in _rdns_cache: return _rdns_cache[key] - result = (False, "") + result = (False, "", "mismatch") try: host = socket.gethostbyaddr(ip)[0].rstrip(".").lower() if any(host.endswith(s) or host == s.lstrip(".") for s in suffixes): @@ -179,13 +229,16 @@ def fcrdns(ip, suffixes): except OSError: pass if ip in addrs: - result = (True, host) + result = (True, host, "ok") else: - result = (False, host + " (no forward match)") + result = (False, host, "mismatch") else: - result = (False, host) + result = (False, host, "mismatch") except OSError: - result = (False, "") + # No PTR at all. This is absence of evidence, not evidence of + # forgery: Meta's IPv6 crawler space, for one, publishes no PTR + # records. Kept distinct so a real mismatch stays visible. + result = (False, "", "no-ptr") _rdns_cache[key] = result return result @@ -268,8 +321,10 @@ all_ips = set().union(*(v[1] for v in per_day.values())) # --- resolve rDNS candidates --------------------------------------------- # One pass over unique addresses, busiest first, capped by RDNS_MAX so a # flood of distinct forgeries cannot stall the report on DNS timeouts. -rdns_ok = Counter() # bot -> hits from forward-confirmed addresses -rdns_bad = Counter() # bot -> hits that failed confirmation +rdns_ok = Counter() # bot -> forward-confirmed by PTR +rdns_mismatch = Counter() # bot -> PTR exists but points elsewhere: forgery +rdns_noptr = Counter() # bot -> no PTR at all: cannot be judged by DNS +rdns_asn = Counter() # bot -> no PTR, but origin ASN is the vendor's rdns_skipped = Counter() # bot -> hits left unresolved by the cap rdns_private = Counter() # bot -> hits from RFC1918/loopback (see below) rdns_ok_ips = {} @@ -289,12 +344,20 @@ if WEEKLY and RDNS_ENABLE: rdns_skipped[bot] += hits continue budget -= 1 - ok, _host = fcrdns(ip, BOT_RDNS[bot]) + ok, _host, why = fcrdns(ip, BOT_RDNS[bot]) if ok: rdns_ok[bot] += hits rdns_ok_ips.setdefault(bot, set()).add(ip) + elif why == "no-ptr": + # Fall back to ASN, but only for vendors whose network is + # single-tenant (see BOT_ASN). Otherwise leave it unknown. + if ASN_ENABLE and bot in BOT_ASN and asn_of(ip) in BOT_ASN[bot]: + rdns_asn[bot] += hits + rdns_ok_ips.setdefault(bot, set()).add(ip) + else: + rdns_noptr[bot] += hits else: - rdns_bad[bot] += hits + rdns_mismatch[bot] += hits socket.setdefaulttimeout(None) if not WEEKLY: @@ -353,31 +416,38 @@ else: lines.append(f" {c:>6} {p}") # --- vendors verified by reverse DNS --------------------------------- - if RDNS_ENABLE and (rdns_ok or rdns_bad or rdns_proxied): + if RDNS_ENABLE and (rdns_ok or rdns_mismatch or rdns_noptr or rdns_asn + or rdns_proxied or rdns_private): lines.append("") lines.append("-" * 62) - lines.append("VERIFIED BY REVERSE DNS (vendors publishing no IP ranges)") + lines.append("VERIFIED BY DNS / ASN (vendors publishing no IP ranges)") lines.append("-" * 62) - lines.append(f" {'bot':<22}{'confirmed':>10}{'failed':>8}{'via CF':>8}" - f"{'private':>8} {'IPs':>4}") - for bot in sorted(set(rdns_ok) | set(rdns_bad) | set(rdns_proxied) | set(rdns_private), - key=lambda b: -(rdns_ok[b] + rdns_bad[b] + rdns_proxied[b] - + rdns_private[b])): - lines.append(f" {bot:<22}{rdns_ok[bot]:>10}{rdns_bad[bot]:>8}" - f"{rdns_proxied[bot]:>8}{rdns_private[bot]:>8}" + lines.append(f" {'bot':<20}{'PTR ok':>7}{'ASN ok':>7}{'FORGED':>8}" + f"{'no-PTR':>7}{'CF':>6}{'priv':>7} {'IPs':>4}") + allbots = (set(rdns_ok) | set(rdns_mismatch) | set(rdns_noptr) + | set(rdns_asn) | set(rdns_proxied) | set(rdns_private)) + for bot in sorted(allbots, key=lambda b: -(rdns_ok[b] + rdns_mismatch[b] + + rdns_noptr[b] + rdns_asn[b] + + rdns_proxied[b] + rdns_private[b])): + lines.append(f" {bot:<20}{rdns_ok[bot]:>7}{rdns_asn[bot]:>7}" + f"{rdns_mismatch[bot]:>8}{rdns_noptr[bot]:>7}" + f"{rdns_proxied[bot]:>6}{rdns_private[bot]:>7}" f" {len(rdns_ok_ips.get(bot, ())):>4}") if rdns_skipped: sk = ", ".join(f"{b} {c}" for b, c in rdns_skipped.most_common(4)) lines.append(f" (unresolved, over CRAWLER_RDNS_MAX={RDNS_MAX}: {sk})") lines.append("") - lines.append(" Forward-confirmed: the address's PTR record ends in a vendor") - lines.append(" domain AND that hostname resolves back to the same address.") - lines.append(" 'failed' means the PTR was absent, pointed elsewhere, or did") - lines.append(" not confirm - i.e. the user-agent is unsupported by DNS.") + lines.append(" PTR ok forward-confirmed: PTR ends in a vendor domain AND") + lines.append(" that hostname resolves back to the same address.") + lines.append(" ASN ok no PTR, but the address sits in an ASN belonging") + lines.append(" solely to the vendor (Meta's AS32934). Only used") + lines.append(" where the ASN sells no compute - never for clouds.") + lines.append(" FORGED a PTR exists and points somewhere else. This is") + lines.append(" the only column that is evidence of forgery.") + lines.append(" no-PTR no PTR record and no usable ASN: unknown, not bad.") if sum(rdns_private.values()): - lines.append(" 'private' means the logged source was an RFC1918 address:") - lines.append(" a proxy or Docker bridge is masking the real client, so") - lines.append(" these cannot be judged either way. See README.") + lines.append(" priv logged source was RFC1918: a proxy or Docker") + lines.append(" bridge masked the client. Unjudgeable. See README.") body = "\n".join(lines) + "\n" subject = f"{SITE} weekly crawler report: {total} requests, {len(all_ips)} IPs" if DRY: