asxpi/traefik-crawlers-statistics
asxpi/traefik-crawlers-statistics
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Separate forged from unverifiable, add ASN check for Meta

Browse source 
The rDNS check had one failure bucket, so "PTR points at a different
network" and "no PTR exists" landed in the same column. Those mean
opposite things. Meta's IPv6 crawler space publishes no PTR records at
all, so every genuine Meta request was reported under a heading that
read as forgery - an accusation the data did not support.

Split the outcome three ways: FORGED (a PTR exists and points
elsewhere - the only column that is evidence of a lie), no-PTR
(nothing to check, unknown), and PTR ok.

Add an ASN fallback for the no-PTR case, restricted to vendors whose
ASN is single-tenant. Meta is the motivating case: it publishes no
range file - their documentation says to email webmasters@meta.com -
and no PTR records, so without this there is nothing to check at all.
AS32934 is Meta's own network and cannot be rented, which is what
makes the match meaningful. This is deliberately not applied to cloud
ASNs: genuine YouBot and genuine Amazonbot both live in AS14618
alongside every EC2 instance a spoofer could rent, so a match there
would prove only that the traffic came from a cloud.

Verified against real traffic: Meta now shows 1 ASN-confirmed request
against 14 forged ones from googleusercontent.com hosts, which is the
distinction the old single column destroyed.

Assisted-by: Claude:opus-5
This commit is contained in:
Sergei Poljanski 2026-08-11 04:52:15 +04:00
commit 43a88182cd
Signed by: asxpi
GPG key ID: 4F8851660FA4121B
2 changed files with 101 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

6
crawler-alert.env.example
View file

@ -47,3 +47,9 @@ CRAWLER_RDNS=1
# Max unique addresses resolved per bot per run. Each costs two DNS
# lookups; the busiest addresses are resolved first.
CRAWLER_RDNS_MAX=400
# ASN fallback for vendors with no PTR records and no published ranges
# (currently Meta / AS32934 only). Used ONLY where the ASN is
# single-tenant; never for cloud ASNs where anyone can rent space.
# Requires `dig`. Set to 0 to disable.
CRAWLER_ASN=1

120
crawler-alert.py
View file

@ -87,6 +87,25 @@ BOT_RDNS = {
# and results are memoised for the run.
RDNS_MAX = int(os.environ.get("CRAWLER_RDNS_MAX", "400"))
RDNS_ENABLE = os.environ.get("CRAWLER_RDNS", "1") != "0"
# --- ASN verification, for single-tenant networks only -------------------
# An ASN match usually proves nothing: genuine YouBot and genuine
# Amazonbot both live in AS14618, which is also every EC2 instance a
# spoofer could rent. It is only evidence when the ASN belongs to the
# vendor exclusively and sells no compute in it. Meta qualifies -
# AS32934 is Meta's own network and cannot be rented - and Meta
# publishes no IP range file (their docs say to email webmasters@meta.com),
# so without this there is nothing to check them against at all.
BOT_ASN = {
"meta-externalagent": {"32934"},
"meta-externalfetcher": {"32934"},
"FacebookBot": {"32934"},
"facebookexternalhit": {"32934"},
}
ASN_ENABLE = os.environ.get("CRAWLER_ASN", "1") != "0"
# Team Cymru's DNS interface: <reversed-addr>.origin[6].asn.cymru.com TXT
CYMRU4 = "origin.asn.cymru.com"
CYMRU6 = "origin6.asn.cymru.com"
CACHE = os.environ.get("CRAWLER_CACHE", "/var/cache/crawler-alert/ranges.json")
CF_URL = "https://www.cloudflare.com/ips-v4"
# claude.com and cloudflare.com sit behind Cloudflare, which 403s the
@ -155,6 +174,37 @@ def in_index(idx_v, addr):
return any(addr in n for n in idx_v.get(key, []))
_asn_cache = {}
def asn_of(ip):
"""Origin ASN for an address via Team Cymru's DNS interface.
Returns the ASN as a string, or "" if it cannot be determined.
Needs `dig`; absence degrades to no ASN evidence, never an error.
"""
if ip in _asn_cache:
return _asn_cache[ip]
asn = ""
try:
a = ipaddress.ip_address(ip)
if a.version == 4:
q = ".".join(reversed(ip.split("."))) + "." + CYMRU4
else:
nibbles = a.exploded.replace(":", "")
q = ".".join(reversed(nibbles)) + "." + CYMRU6
out = subprocess.run(["dig", "+short", "+time=3", "+tries=1", q, "TXT"],
capture_output=True, text=True, timeout=12).stdout
# "32934 | 2a03:2880:f814::/48 | IE | ripencc | 2011-08-22"
first = out.strip().strip('"').split("|")[0].strip()
if first.split(" ")[0].isdigit():
asn = first.split(" ")[0]
except (OSError, ValueError, subprocess.SubprocessError):
pass
_asn_cache[ip] = asn
return asn
_rdns_cache = {}
@ -168,7 +218,7 @@ def fcrdns(ip, suffixes):
key = (ip, suffixes)
if key in _rdns_cache:
return _rdns_cache[key]
result = (False, "")
result = (False, "", "mismatch")
try:
host = socket.gethostbyaddr(ip)[0].rstrip(".").lower()
if any(host.endswith(s) or host == s.lstrip(".") for s in suffixes):
@ -179,13 +229,16 @@ def fcrdns(ip, suffixes):
except OSError:
pass
if ip in addrs:
result = (True, host)
result = (True, host, "ok")
else:
result = (False, host + " (no forward match)")
result = (False, host, "mismatch")
else:
result = (False, host)
result = (False, host, "mismatch")
except OSError:
result = (False, "")
# No PTR at all. This is absence of evidence, not evidence of
# forgery: Meta's IPv6 crawler space, for one, publishes no PTR
# records. Kept distinct so a real mismatch stays visible.
result = (False, "", "no-ptr")
_rdns_cache[key] = result
return result
@ -268,8 +321,10 @@ all_ips = set().union(*(v[1] for v in per_day.values()))
# --- resolve rDNS candidates ---------------------------------------------
# One pass over unique addresses, busiest first, capped by RDNS_MAX so a
# flood of distinct forgeries cannot stall the report on DNS timeouts.
rdns_ok = Counter() # bot -> hits from forward-confirmed addresses
rdns_bad = Counter() # bot -> hits that failed confirmation
rdns_ok = Counter() # bot -> forward-confirmed by PTR
rdns_mismatch = Counter() # bot -> PTR exists but points elsewhere: forgery
rdns_noptr = Counter() # bot -> no PTR at all: cannot be judged by DNS
rdns_asn = Counter() # bot -> no PTR, but origin ASN is the vendor's
rdns_skipped = Counter() # bot -> hits left unresolved by the cap
rdns_private = Counter() # bot -> hits from RFC1918/loopback (see below)
rdns_ok_ips = {}
@ -289,12 +344,20 @@ if WEEKLY and RDNS_ENABLE:
rdns_skipped[bot] += hits
continue
budget -= 1
ok, _host = fcrdns(ip, BOT_RDNS[bot])
ok, _host, why = fcrdns(ip, BOT_RDNS[bot])
if ok:
rdns_ok[bot] += hits
rdns_ok_ips.setdefault(bot, set()).add(ip)
elif why == "no-ptr":
# Fall back to ASN, but only for vendors whose network is
# single-tenant (see BOT_ASN). Otherwise leave it unknown.
if ASN_ENABLE and bot in BOT_ASN and asn_of(ip) in BOT_ASN[bot]:
rdns_asn[bot] += hits
rdns_ok_ips.setdefault(bot, set()).add(ip)
else:
rdns_noptr[bot] += hits
else:
rdns_bad[bot] += hits
rdns_mismatch[bot] += hits
socket.setdefaulttimeout(None)
if not WEEKLY:
@ -353,31 +416,38 @@ else:
lines.append(f" {c:>6} {p}")
# --- vendors verified by reverse DNS ---------------------------------
if RDNS_ENABLE and (rdns_ok or rdns_bad or rdns_proxied):
if RDNS_ENABLE and (rdns_ok or rdns_mismatch or rdns_noptr or rdns_asn
or rdns_proxied or rdns_private):
lines.append("")
lines.append("-" * 62)
lines.append("VERIFIED BY REVERSE DNS (vendors publishing no IP ranges)")
lines.append("VERIFIED BY DNS / ASN (vendors publishing no IP ranges)")
lines.append("-" * 62)
lines.append(f" {'bot':<22}{'confirmed':>10}{'failed':>8}{'via CF':>8}"
f"{'private':>8} {'IPs':>4}")
for bot in sorted(set(rdns_ok) | set(rdns_bad) | set(rdns_proxied) | set(rdns_private),
key=lambda b: -(rdns_ok[b] + rdns_bad[b] + rdns_proxied[b]
+ rdns_private[b])):
lines.append(f" {bot:<22}{rdns_ok[bot]:>10}{rdns_bad[bot]:>8}"
f"{rdns_proxied[bot]:>8}{rdns_private[bot]:>8}"
lines.append(f" {'bot':<20}{'PTR ok':>7}{'ASN ok':>7}{'FORGED':>8}"
f"{'no-PTR':>7}{'CF':>6}{'priv':>7} {'IPs':>4}")
allbots = (set(rdns_ok) | set(rdns_mismatch) | set(rdns_noptr)
| set(rdns_asn) | set(rdns_proxied) | set(rdns_private))
for bot in sorted(allbots, key=lambda b: -(rdns_ok[b] + rdns_mismatch[b]
+ rdns_noptr[b] + rdns_asn[b]
+ rdns_proxied[b] + rdns_private[b])):
lines.append(f" {bot:<20}{rdns_ok[bot]:>7}{rdns_asn[bot]:>7}"
f"{rdns_mismatch[bot]:>8}{rdns_noptr[bot]:>7}"
f"{rdns_proxied[bot]:>6}{rdns_private[bot]:>7}"
f" {len(rdns_ok_ips.get(bot, ())):>4}")
if rdns_skipped:
sk = ", ".join(f"{b} {c}" for b, c in rdns_skipped.most_common(4))
lines.append(f" (unresolved, over CRAWLER_RDNS_MAX={RDNS_MAX}: {sk})")
lines.append("")
lines.append(" Forward-confirmed: the address's PTR record ends in a vendor")
lines.append(" domain AND that hostname resolves back to the same address.")
lines.append(" 'failed' means the PTR was absent, pointed elsewhere, or did")
lines.append(" not confirm - i.e. the user-agent is unsupported by DNS.")
lines.append(" PTR ok forward-confirmed: PTR ends in a vendor domain AND")
lines.append(" that hostname resolves back to the same address.")
lines.append(" ASN ok no PTR, but the address sits in an ASN belonging")
lines.append(" solely to the vendor (Meta's AS32934). Only used")
lines.append(" where the ASN sells no compute - never for clouds.")
lines.append(" FORGED a PTR exists and points somewhere else. This is")
lines.append(" the only column that is evidence of forgery.")
lines.append(" no-PTR no PTR record and no usable ASN: unknown, not bad.")
if sum(rdns_private.values()):
lines.append(" 'private' means the logged source was an RFC1918 address:")
lines.append(" a proxy or Docker bridge is masking the real client, so")
lines.append(" these cannot be judged either way. See README.")
lines.append(" priv logged source was RFC1918: a proxy or Docker")
lines.append(" bridge masked the client. Unjudgeable. See README.")
body = "\n".join(lines) + "\n"
subject = f"{SITE} weekly crawler report: {total} requests, {len(all_ips)} IPs"
if DRY: