asxpi/traefik-crawlers-statistics
asxpi/traefik-crawlers-statistics
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Make the tool reusable on other instances

Browse source 
Everything was hardcoded for one deployment: hostname in mail subjects,
recipient address, log path, and a router name of "forgejo@docker" that
also appeared inside the user-agent regex. That last one fails silently
rather than loudly - on any instance whose Traefik router is named
something else, the regex matches nothing, every user-agent is dropped,
and the report claims zero bot traffic instead of erroring.

Move configuration to environment variables read from
/etc/crawler-alert.env, with defaults that suit a plain Traefik host.
Only CRAWLER_RECIPIENT is required, and the script now refuses to run
without it rather than mailing into the void. Anchor the user-agent
regex on the request counter instead of the router name.

SMTP can now be configured directly via CRAWLER_SMTP_*, so the tool no
longer requires Forgejo in Docker; borrowing credentials from app.ini
stays the default since it avoids a second copy of the password.
Unauthenticated and non-TLS local relays are handled.

Document the Traefik access log configuration in
traefik-accesslog.yml. This is the one real prerequisite: Traefik drops
all headers by default, so without an explicit User-Agent: keep there
is nothing to analyse. Includes the field layout the parser expects,
logrotate config, and the forwardedHeaders setup needed to recover real
client IPs from behind Cloudflare.

Rewrite README for someone arriving without context, and add a 0BSD
LICENSE so the code can actually be reused.

Assisted-by: Claude:opus-5
This commit is contained in:
Sergei Poljanski 2026-08-11 02:49:51 +04:00
commit 5860cbee9a
Signed by: asxpi
GPG key ID: 4F8851660FA4121B
7 changed files with 343 additions and 85 deletions
Download patch file Download diff file Expand all files Collapse all files

99
crawler-alert.py
View file

@ -1,23 +1,47 @@
#!/usr/bin/env python3
"""Forgejo crawler watch on git.asxp.io. Two modes:
(default) daily threshold alert: mail only if yesterday > 100k requests
--weekly Monday summary: always mail stats for the past 7 days
"""Crawler watch for a Traefik-fronted Forgejo (or any Traefik site).
Separates real AI crawlers from traffic that only claims to be, by
checking the source IP against each vendor's published range list.
Modes:
(default) daily threshold alert: mail only if yesterday exceeded
CRAWLER_THRESHOLD requests
--weekly summary: always mail stats for the past 7 days
--dry-run print instead of mailing (combinable with --weekly)
Set up 2026-07-15; timers: crawler-alert.timer, crawler-weekly.timer."""
Configuration is by environment variable; see README.md. The only
value with no sensible default is CRAWLER_RECIPIENT.
"""
import configparser, glob, gzip, ipaddress, json, os, re, smtplib, subprocess, sys, urllib.request
from collections import Counter
from datetime import date, timedelta
from email.mime.text import MIMEText
THRESHOLD = 100_000
RECIPIENT = "me@asxp.io"
# --- configuration ------------------------------------------------------
THRESHOLD = int(os.environ.get("CRAWLER_THRESHOLD", "100000"))
RECIPIENT = os.environ.get("CRAWLER_RECIPIENT", "")
SITE = os.environ.get("CRAWLER_SITE", "this site")
LOG_GLOB = os.environ.get("CRAWLER_LOG", "/var/log/traefik/access.log*")
# Traefik router name to filter on, e.g. "forgejo@docker". Empty = all
# traffic reaching Traefik, which is what you want for a single-site host.
ROUTER = os.environ.get("CRAWLER_ROUTER", "")
# Container to read SMTP settings from (Forgejo/Gitea app.ini [mailer]).
# Set CRAWLER_SMTP_* instead to configure SMTP directly.
MAIL_CONTAINER = os.environ.get("CRAWLER_MAIL_CONTAINER", "forgejo")
MAIL_INI = os.environ.get("CRAWLER_MAIL_INI", "/data/gitea/conf/app.ini")
WEEKLY = "--weekly" in sys.argv
DRY = "--dry-run" in sys.argv
if not RECIPIENT and not DRY:
sys.exit("CRAWLER_RECIPIENT is not set (no --dry-run either); refusing "
"to run. See README.md.")
# --- verified-crawler support -------------------------------------------
# Vendors publishing machine-readable IP ranges. A UA token alone proves
# nothing: as of Aug 2026 ~77% of AI-bot-labelled traffic here came from
# hosts outside these ranges (mostly GCP). Only IP-verified hits count.
# nothing: on the instance this was written for, ~83% of AI-bot-labelled
# traffic came from hosts outside these ranges. Only IP-verified hits count.
RANGE_SOURCES = {
"openai": ["https://openai.com/chatgpt-user.json",
"https://openai.com/gptbot.json",
@ -38,11 +62,11 @@ BOT_VENDOR = {
"Applebot-Extended": "apple", "Applebot": "apple",
"Googlebot": "google", "Google-Extended": "google", "GoogleOther": "google",
}
CACHE = "/var/cache/crawler-alert/ranges.json"
CACHE = os.environ.get("CRAWLER_CACHE", "/var/cache/crawler-alert/ranges.json")
CF_URL = "https://www.cloudflare.com/ips-v4"
# claude.com and cloudflare.com sit behind Cloudflare, which 403s the
# default Python-urllib UA. Send a real one.
UA = "Mozilla/5.0 (compatible; crawler-alert/1.0; +https://git.asxp.io)"
UA = "Mozilla/5.0 (compatible; crawler-alert/1.0; +https://codeberg.org/)"
def fetch(url, timeout=20):
@ -118,11 +142,11 @@ spoofed = Counter() # bot -> UA claimed, IP outside vendor ranges
proxied = Counter() # bot -> arrived via Cloudflare, unverifiable
ver_paths = Counter() # paths fetched by verified bots
ver_ips = {} # bot -> set of verified source IPs
for path in sorted(glob.glob("/opt/traefik/logs/access.log*")):
for path in sorted(glob.glob(LOG_GLOB)):
opener = gzip.open if path.endswith(".gz") else open
with opener(path, "rt", errors="replace") as f:
for line in f:
if "forgejo@docker" not in line:
if ROUTER and ROUTER not in line:
continue
for s in stamps:
if s in line:
@ -131,7 +155,10 @@ for path in sorted(glob.glob("/opt/traefik/logs/access.log*")):
parts = line.split()
if len(parts) > 6 and parts[6].count("/") >= 2:
repos["/".join(parts[6].split("/")[1:3])] += 1
ua = re.search(r"\" \"([^\"]*)\" \d+ \"forgejo", line)
# Traefik CLF: ... "<referer>" "<user-agent>" <n> "<router>" ...
# Anchor on the request counter so this works whatever
# the router is called.
ua = re.search(r"\"[^\"]*\" \"([^\"]*)\" \d+ \"", line)
if ua:
u = ua.group(1)
m = re.search(r"(bot|crawler|spider|scrapy|externalagent|gpt|claude|perplexity)[\w./-]*", u, re.I)
@ -163,15 +190,15 @@ all_ips = set().union(*(v[1] for v in per_day.values()))
if not WEEKLY:
s0 = stamps[0]
count, ips = per_day[s0][0], len(per_day[s0][1])
print(f"{s0}: {count} forgejo requests, {ips} unique IPs (threshold {THRESHOLD})")
print(f"{s0}: {count} requests, {ips} unique IPs (threshold {THRESHOLD})")
if DRY or count <= THRESHOLD:
sys.exit(0)
subject = f"git.asxp.io: {count} requests on {s0} (>100k/day threshold)"
subject = f"{SITE}: {count} requests on {s0} (over {THRESHOLD}/day threshold)"
body = (f"Forgejo served {count} requests from {ips} unique IPs on {s0} - "
f"above the {THRESHOLD}/day watch threshold.\n\n"
"Time to consider Anubis (PoW challenge) in front of Forgejo.\n")
else:
lines = [f"git.asxp.io weekly crawler report ({stamps[0]} - {stamps[-1]})", ""]
lines = [f"{SITE} weekly crawler report ({stamps[0]} - {stamps[-1]})", ""]
lines.append(f"Total: {total} requests, {len(all_ips)} unique IPs")
lines.append(f"Daily threshold alert fires above {THRESHOLD} req/day (none = quiet week)")
lines.append("")
@ -215,20 +242,34 @@ else:
for p, c in ver_paths.most_common(8):
lines.append(f" {c:>6} {p}")
body = "\n".join(lines) + "\n"
subject = f"git.asxp.io weekly crawler report: {total} requests, {len(all_ips)} IPs"
subject = f"{SITE} weekly crawler report: {total} requests, {len(all_ips)} IPs"
if DRY:
print(subject); print(); print(body)
sys.exit(0)
ini = subprocess.run(["docker","exec","forgejo","cat","/data/gitea/conf/app.ini"],
capture_output=True, text=True).stdout
cp = configparser.ConfigParser(interpolation=None, strict=False)
cp.read_string("[DEFAULT]\n" + ini)
m = cp["mailer"]
proto = m.get("PROTOCOL", "smtps").strip()
addr, port = m.get("SMTP_ADDR").strip(), int(m.get("SMTP_PORT", "465").strip())
user, passwd = m.get("USER").strip(), m.get("PASSWD").strip().strip("`\"")
sender = m.get("FROM", user).strip()
# SMTP settings: use CRAWLER_SMTP_* if given, otherwise borrow them from
# a Forgejo/Gitea app.ini [mailer] section so there is no second copy of
# the credentials to keep in sync.
if os.environ.get("CRAWLER_SMTP_ADDR"):
addr = os.environ["CRAWLER_SMTP_ADDR"]
port = int(os.environ.get("CRAWLER_SMTP_PORT", "587"))
user = os.environ.get("CRAWLER_SMTP_USER", "")
passwd = os.environ.get("CRAWLER_SMTP_PASSWORD", "")
sender = os.environ.get("CRAWLER_SMTP_FROM", user or RECIPIENT)
proto = os.environ.get("CRAWLER_SMTP_PROTOCOL", "smtp+starttls")
else:
ini = subprocess.run(["docker", "exec", MAIL_CONTAINER, "cat", MAIL_INI],
capture_output=True, text=True).stdout
if not ini.strip():
sys.exit(f"could not read {MAIL_INI} from container "
f"'{MAIL_CONTAINER}' and CRAWLER_SMTP_ADDR is unset")
cp = configparser.ConfigParser(interpolation=None, strict=False)
cp.read_string("[DEFAULT]\n" + ini)
m = cp["mailer"]
proto = m.get("PROTOCOL", "smtps").strip()
addr, port = m.get("SMTP_ADDR").strip(), int(m.get("SMTP_PORT", "465").strip())
user, passwd = m.get("USER", "").strip(), m.get("PASSWD", "").strip().strip("`\"")
sender = m.get("FROM", user).strip()
msg = MIMEText(body)
msg["Subject"], msg["From"], msg["To"] = subject, sender, RECIPIENT
@ -236,8 +277,10 @@ if proto == "smtps":
s = smtplib.SMTP_SSL(addr, port, timeout=30)
else:
s = smtplib.SMTP(addr, port, timeout=30)
s.starttls()
s.login(user, passwd)
if proto != "smtp": # plain smtp = no TLS (local relay)
s.starttls()
if user: # unauthenticated local relays exist
s.login(user, passwd)
s.sendmail(sender, [RECIPIENT], msg.as_string())
s.quit()
print(f"mailed to {RECIPIENT}")