Separate forged from unverifiable, add ASN check for Meta

The rDNS check had one failure bucket, so "PTR points at a different
network" and "no PTR exists" landed in the same column. Those mean
opposite things. Meta's IPv6 crawler space publishes no PTR records at
all, so every genuine Meta request was reported under a heading that
read as forgery - an accusation the data did not support.

Split the outcome three ways: FORGED (a PTR exists and points
elsewhere - the only column that is evidence of a lie), no-PTR
(nothing to check, unknown), and PTR ok.

Add an ASN fallback for the no-PTR case, restricted to vendors whose
ASN is single-tenant. Meta is the motivating case: it publishes no
range file - their documentation says to email webmasters@meta.com -
and no PTR records, so without this there is nothing to check at all.
AS32934 is Meta's own network and cannot be rented, which is what
makes the match meaningful. This is deliberately not applied to cloud
ASNs: genuine YouBot and genuine Amazonbot both live in AS14618
alongside every EC2 instance a spoofer could rent, so a match there
would prove only that the traffic came from a cloud.

Verified against real traffic: Meta now shows 1 ASN-confirmed request
against 14 forged ones from googleusercontent.com hosts, which is the
distinction the old single column destroyed.

Assisted-by: Claude:opus-5

crawler-alert.env.example

@@ -47,3 +47,9 @@ CRAWLER_RDNS=1
 # Max unique addresses resolved per bot per run. Each costs two DNS
 # lookups; the busiest addresses are resolved first.
 CRAWLER_RDNS_MAX=400
+
+# ASN fallback for vendors with no PTR records and no published ranges
+# (currently Meta / AS32934 only). Used ONLY where the ASN is
+# single-tenant; never for cloud ASNs where anyone can rent space.
+# Requires `dig`. Set to 0 to disable.
+CRAWLER_ASN=1

crawler-alert.py

@@ -87,6 +87,25 @@ BOT_RDNS = {
 # and results are memoised for the run.
 RDNS_MAX = int(os.environ.get("CRAWLER_RDNS_MAX", "400"))
 RDNS_ENABLE = os.environ.get("CRAWLER_RDNS", "1") != "0"
+
+# --- ASN verification, for single-tenant networks only -------------------
+# An ASN match usually proves nothing: genuine YouBot and genuine
+# Amazonbot both live in AS14618, which is also every EC2 instance a
+# spoofer could rent. It is only evidence when the ASN belongs to the
+# vendor exclusively and sells no compute in it. Meta qualifies -
+# AS32934 is Meta's own network and cannot be rented - and Meta
+# publishes no IP range file (their docs say to email webmasters@meta.com),
+# so without this there is nothing to check them against at all.
+BOT_ASN = {
+    "meta-externalagent":   {"32934"},
+    "meta-externalfetcher": {"32934"},
+    "FacebookBot":          {"32934"},
+    "facebookexternalhit":  {"32934"},
+}
+ASN_ENABLE = os.environ.get("CRAWLER_ASN", "1") != "0"
+# Team Cymru's DNS interface: <reversed-addr>.origin[6].asn.cymru.com TXT
+CYMRU4 = "origin.asn.cymru.com"
+CYMRU6 = "origin6.asn.cymru.com"
 CACHE = os.environ.get("CRAWLER_CACHE", "/var/cache/crawler-alert/ranges.json")
 CF_URL = "https://www.cloudflare.com/ips-v4"
 # claude.com and cloudflare.com sit behind Cloudflare, which 403s the
@@ -155,6 +174,37 @@ def in_index(idx_v, addr):
     return any(addr in n for n in idx_v.get(key, []))
 
 
+_asn_cache = {}
+
+
+def asn_of(ip):
+    """Origin ASN for an address via Team Cymru's DNS interface.
+
+    Returns the ASN as a string, or "" if it cannot be determined.
+    Needs `dig`; absence degrades to no ASN evidence, never an error.
+    """
+    if ip in _asn_cache:
+        return _asn_cache[ip]
+    asn = ""
+    try:
+        a = ipaddress.ip_address(ip)
+        if a.version == 4:
+            q = ".".join(reversed(ip.split("."))) + "." + CYMRU4
+        else:
+            nibbles = a.exploded.replace(":", "")
+            q = ".".join(reversed(nibbles)) + "." + CYMRU6
+        out = subprocess.run(["dig", "+short", "+time=3", "+tries=1", q, "TXT"],
+                             capture_output=True, text=True, timeout=12).stdout
+        # "32934 | 2a03:2880:f814::/48 | IE | ripencc | 2011-08-22"
+        first = out.strip().strip('"').split("|")[0].strip()
+        if first.split(" ")[0].isdigit():
+            asn = first.split(" ")[0]
+    except (OSError, ValueError, subprocess.SubprocessError):
+        pass
+    _asn_cache[ip] = asn
+    return asn
+
+
 _rdns_cache = {}
 
 
@@ -168,7 +218,7 @@ def fcrdns(ip, suffixes):
     key = (ip, suffixes)
     if key in _rdns_cache:
         return _rdns_cache[key]
-    result = (False, "")
+    result = (False, "", "mismatch")
     try:
         host = socket.gethostbyaddr(ip)[0].rstrip(".").lower()
         if any(host.endswith(s) or host == s.lstrip(".") for s in suffixes):
@@ -179,13 +229,16 @@ def fcrdns(ip, suffixes):
             except OSError:
                 pass
             if ip in addrs:
-                result = (True, host)
+                result = (True, host, "ok")
             else:
-                result = (False, host + " (no forward match)")
+                result = (False, host, "mismatch")
         else:
-            result = (False, host)
+            result = (False, host, "mismatch")
     except OSError:
-        result = (False, "")
+        # No PTR at all. This is absence of evidence, not evidence of
+        # forgery: Meta's IPv6 crawler space, for one, publishes no PTR
+        # records. Kept distinct so a real mismatch stays visible.
+        result = (False, "", "no-ptr")
     _rdns_cache[key] = result
     return result
 
@@ -268,8 +321,10 @@ all_ips = set().union(*(v[1] for v in per_day.values()))
 # --- resolve rDNS candidates ---------------------------------------------
 # One pass over unique addresses, busiest first, capped by RDNS_MAX so a
 # flood of distinct forgeries cannot stall the report on DNS timeouts.
-rdns_ok = Counter()       # bot -> hits from forward-confirmed addresses
+rdns_ok = Counter()       # bot -> forward-confirmed by PTR
-rdns_bad = Counter()      # bot -> hits that failed confirmation
+rdns_mismatch = Counter() # bot -> PTR exists but points elsewhere: forgery
+rdns_noptr = Counter()    # bot -> no PTR at all: cannot be judged by DNS
+rdns_asn = Counter()      # bot -> no PTR, but origin ASN is the vendor's
 rdns_skipped = Counter()  # bot -> hits left unresolved by the cap
 rdns_private = Counter()  # bot -> hits from RFC1918/loopback (see below)
 rdns_ok_ips = {}
@@ -289,12 +344,20 @@ if WEEKLY and RDNS_ENABLE:
                 rdns_skipped[bot] += hits
                 continue
             budget -= 1
-            ok, _host = fcrdns(ip, BOT_RDNS[bot])
+            ok, _host, why = fcrdns(ip, BOT_RDNS[bot])
             if ok:
                 rdns_ok[bot] += hits
                 rdns_ok_ips.setdefault(bot, set()).add(ip)
+            elif why == "no-ptr":
+                # Fall back to ASN, but only for vendors whose network is
+                # single-tenant (see BOT_ASN). Otherwise leave it unknown.
+                if ASN_ENABLE and bot in BOT_ASN and asn_of(ip) in BOT_ASN[bot]:
+                    rdns_asn[bot] += hits
+                    rdns_ok_ips.setdefault(bot, set()).add(ip)
                 else:
-                rdns_bad[bot] += hits
+                    rdns_noptr[bot] += hits
+            else:
+                rdns_mismatch[bot] += hits
     socket.setdefaulttimeout(None)
 
 if not WEEKLY:
@@ -353,31 +416,38 @@ else:
                 lines.append(f"    {c:>6}  {p}")
 
     # --- vendors verified by reverse DNS ---------------------------------
-    if RDNS_ENABLE and (rdns_ok or rdns_bad or rdns_proxied):
+    if RDNS_ENABLE and (rdns_ok or rdns_mismatch or rdns_noptr or rdns_asn
+                        or rdns_proxied or rdns_private):
         lines.append("")
         lines.append("-" * 62)
-        lines.append("VERIFIED BY REVERSE DNS (vendors publishing no IP ranges)")
+        lines.append("VERIFIED BY DNS / ASN (vendors publishing no IP ranges)")
         lines.append("-" * 62)
-        lines.append(f"  {'bot':<22}{'confirmed':>10}{'failed':>8}{'via CF':>8}"
+        lines.append(f"  {'bot':<20}{'PTR ok':>7}{'ASN ok':>7}{'FORGED':>8}"
-                     f"{'private':>8}  {'IPs':>4}")
+                     f"{'no-PTR':>7}{'CF':>6}{'priv':>7}  {'IPs':>4}")
-        for bot in sorted(set(rdns_ok) | set(rdns_bad) | set(rdns_proxied) | set(rdns_private),
+        allbots = (set(rdns_ok) | set(rdns_mismatch) | set(rdns_noptr)
-                          key=lambda b: -(rdns_ok[b] + rdns_bad[b] + rdns_proxied[b]
+                   | set(rdns_asn) | set(rdns_proxied) | set(rdns_private))
-                                          + rdns_private[b])):
+        for bot in sorted(allbots, key=lambda b: -(rdns_ok[b] + rdns_mismatch[b]
-            lines.append(f"  {bot:<22}{rdns_ok[bot]:>10}{rdns_bad[bot]:>8}"
+                                                   + rdns_noptr[b] + rdns_asn[b]
-                         f"{rdns_proxied[bot]:>8}{rdns_private[bot]:>8}"
+                                                   + rdns_proxied[b] + rdns_private[b])):
+            lines.append(f"  {bot:<20}{rdns_ok[bot]:>7}{rdns_asn[bot]:>7}"
+                         f"{rdns_mismatch[bot]:>8}{rdns_noptr[bot]:>7}"
+                         f"{rdns_proxied[bot]:>6}{rdns_private[bot]:>7}"
                          f"  {len(rdns_ok_ips.get(bot, ())):>4}")
         if rdns_skipped:
             sk = ", ".join(f"{b} {c}" for b, c in rdns_skipped.most_common(4))
             lines.append(f"  (unresolved, over CRAWLER_RDNS_MAX={RDNS_MAX}: {sk})")
         lines.append("")
-        lines.append("  Forward-confirmed: the address's PTR record ends in a vendor")
+        lines.append("  PTR ok  forward-confirmed: PTR ends in a vendor domain AND")
-        lines.append("  domain AND that hostname resolves back to the same address.")
+        lines.append("          that hostname resolves back to the same address.")
-        lines.append("  'failed' means the PTR was absent, pointed elsewhere, or did")
+        lines.append("  ASN ok  no PTR, but the address sits in an ASN belonging")
-        lines.append("  not confirm - i.e. the user-agent is unsupported by DNS.")
+        lines.append("          solely to the vendor (Meta's AS32934). Only used")
+        lines.append("          where the ASN sells no compute - never for clouds.")
+        lines.append("  FORGED  a PTR exists and points somewhere else. This is")
+        lines.append("          the only column that is evidence of forgery.")
+        lines.append("  no-PTR  no PTR record and no usable ASN: unknown, not bad.")
         if sum(rdns_private.values()):
-            lines.append("  'private' means the logged source was an RFC1918 address:")
+            lines.append("  priv    logged source was RFC1918: a proxy or Docker")
-            lines.append("  a proxy or Docker bridge is masking the real client, so")
+            lines.append("          bridge masked the client. Unjudgeable. See README.")
-            lines.append("  these cannot be judged either way. See README.")
     body = "\n".join(lines) + "\n"
     subject = f"{SITE} weekly crawler report: {total} requests, {len(all_ips)} IPs"
     if DRY: