From 3b80ddeba0898825c70ee8f1ad683b2adad7ea03 Mon Sep 17 00:00:00 2001
From: Sergei Poljanski <me@asxp.io>
Date: Tue, 11 Aug 2026 05:22:14 +0400
Subject: [PATCH] Include DNS/ASN-verified crawlers in the path breakdown

The path breakdown was populated only inside the IP-range branch, so
it silently covered one of the two verification methods. Vendors
confirmed by PTR or ASN - YouBot, Amazonbot, Meta - were counted in
their own table but contributed nothing to the only output that shows
what crawlers actually fetch. On the sample instance that hid roughly
40% of the verified requests from the path list.

Collect paths per address during the log pass and merge them once the
address is confirmed, whichever method confirms it. Private-source
hits are excluded, since those addresses are never judged either way.

Move the section to the end of the report and retitle it: it now spans
both tables above, so placing it under the IP-range table implied a
narrower scope than it has.

Assisted-by: Claude:opus-5
---
 crawler-alert.py | 29 +++++++++++++++++++++++------
 1 file changed, 23 insertions(+), 6 deletions(-)

diff --git a/crawler-alert.py b/crawler-alert.py
index d3557f8..51af5f5 100755
--- a/crawler-alert.py
+++ b/crawler-alert.py
@@ -258,6 +258,10 @@ ver_ips = {}              # bot -> set of verified source IPs
 # rDNS-verified vendors: {bot: {ip: hits}}, resolved after the log pass
 rdns_hits = {b: Counter() for b in BOT_RDNS}
 rdns_proxied = Counter()
+# {bot: {ip: Counter(path)}} - kept so that once an address is confirmed
+# by DNS/ASN its paths can join the verified-path breakdown, which
+# otherwise only ever showed IP-range-verified vendors.
+rdns_paths = {b: {} for b in BOT_RDNS}
 for path in sorted(glob.glob(LOG_GLOB)):
     opener = gzip.open if path.endswith(".gz") else open
     with opener(path, "rt", errors="replace") as f:
@@ -313,6 +317,9 @@ for path in sorted(glob.glob(LOG_GLOB)):
                                     rdns_proxied[rb] += 1
                                 else:
                                     rdns_hits[rb][parts[0]] += 1
+                                    if len(parts) > 6 and not a2.is_private:
+                                        rdns_paths[rb].setdefault(
+                                            parts[0], Counter())[parts[6][:60]] += 1
                     break
 
 total = sum(v[0] for v in per_day.values())
@@ -345,19 +352,24 @@ if WEEKLY and RDNS_ENABLE:
                 continue
             budget -= 1
             ok, _host, why = fcrdns(ip, BOT_RDNS[bot])
+            confirmed = False
             if ok:
                 rdns_ok[bot] += hits
                 rdns_ok_ips.setdefault(bot, set()).add(ip)
+                confirmed = True
             elif why == "no-ptr":
                 # Fall back to ASN, but only for vendors whose network is
                 # single-tenant (see BOT_ASN). Otherwise leave it unknown.
                 if ASN_ENABLE and bot in BOT_ASN and asn_of(ip) in BOT_ASN[bot]:
                     rdns_asn[bot] += hits
                     rdns_ok_ips.setdefault(bot, set()).add(ip)
+                    confirmed = True
                 else:
                     rdns_noptr[bot] += hits
             else:
                 rdns_mismatch[bot] += hits
+            if confirmed:
+                ver_paths.update(rdns_paths.get(bot, {}).get(ip, ()))
     socket.setdefaulttimeout(None)
 
 if not WEEKLY:
@@ -409,12 +421,6 @@ else:
         lines.append(f"  {claimed} requests claimed an AI-bot identity; {tv} verified ({pct:.1f}%).")
         lines.append(f"  {ts} failed IP verification (spoofed). {tp} arrived via Cloudflare")
         lines.append("  and cannot be verified by IP - not counted either way.")
-        if ver_paths:
-            lines.append("")
-            lines.append("  Top paths fetched by verified crawlers:")
-            for p, c in ver_paths.most_common(8):
-                lines.append(f"    {c:>6}  {p}")
-
     # --- vendors verified by reverse DNS ---------------------------------
     if RDNS_ENABLE and (rdns_ok or rdns_mismatch or rdns_noptr or rdns_asn
                         or rdns_proxied or rdns_private):
@@ -448,6 +454,17 @@ else:
         if sum(rdns_private.values()):
             lines.append("  priv    logged source was RFC1918: a proxy or Docker")
             lines.append("          bridge masked the client. Unjudgeable. See README.")
+
+    # --- what the verified crawlers actually fetched ---------------------
+    # Last, because it spans both tables above: an address only counts here
+    # once it has been confirmed, whether by IP range, PTR or ASN.
+    if ver_paths:
+        lines.append("")
+        lines.append("=" * 62)
+        lines.append("TOP PATHS FETCHED BY VERIFIED CRAWLERS (all methods)")
+        lines.append("=" * 62)
+        for p, c in ver_paths.most_common(12):
+            lines.append(f"  {c:>6}  {p}")
     body = "\n".join(lines) + "\n"
     subject = f"{SITE} weekly crawler report: {total} requests, {len(all_ips)} IPs"
     if DRY: