diff --git a/ansible/inventory/group_vars/all/vault.yaml b/ansible/inventory/group_vars/all/vault.yaml index 4b7cd8c..7f73a5b 100644 --- a/ansible/inventory/group_vars/all/vault.yaml +++ b/ansible/inventory/group_vars/all/vault.yaml @@ -1,87 +1,106 @@ $ANSIBLE_VAULT;1.1;AES256 -62313564376664646635373735353930353765633834323532646230633033393738376335313666 -3537393466636230353539313938656537333665636136640a653338306635333331383932663965 -39643137366165623666636135646638623830633563313133326231623030643665633062616336 -3035386162363566660a373666353366393034306135373830356166373861633035346637333339 -32346466363162353663663966323635346338333265303565386166373837626437356632383162 -64666536316438346464666165616437376230343763343738363738373130363833393730303138 -62393736326562313230333331633232363362613839393330353133316536343235383464303634 -34396163626530383564396637336365633033373461623233633630626237336231333466346631 -37323439636239663861653065663532343730313130663663626361353835356662613232663062 -62386130313533613430303035653430313866623335303134383738316430346262633639346334 -63326333653761613239663932306135366236376335643733366535666430633938613966386164 -61633137383866396533323831666462346438633830376561623431333666313531623133656230 -30343862383534623062343537326463623731323435363837383038316539376131363534366164 -30323666663338613466626534376562613539383563633765626238633763396235356639393831 -35636138356435376132336365666363633965623965343165303261323831363366666164373061 -31656239356664623038376231363765386664336135373637383165653966373035353836363264 -39366161646334376430623261353230653930623966616535303565383962633339323563363666 -30343834323430333562666230313334366339313838643662376437323236633761386333373336 -36653132393564653431396662613936636639653231646365353836633830616430323430363561 -33313130313633356539303363613731383930623937643461613665626437623365303766353862 -34386566396530343638313334306530633962653235623561326234363639313233666332616130 -62663235663863666230653363663863663031323563383235356465626463303834623464383835 -61626463343638626530376564343333633738313937326662643133633364353232643135666462 -66363965663436303638363535363332363033313766313335393235363230666234643764653732 -62653261656332326461363961353863346563323638326161656331323538643934383665393536 -37323265396662646163363066643237656235616462373337633338623661363037316131323762 -62653562626237316139363833633039636365643438656230656134363634393139386334376431 -36373534326439323130336431303531366565653263323364323139383632643436613839326434 -35643136363664626630346436656562626363633864356132393035343037633632366436393866 -65616363343062656336303463313431363333336365613632393163356531323838333062613664 -65393430313732346364323736333361363363396433313666636433626139353766346165363637 -34616634313833393162393738376466613338616433393235663930663630356561643237663063 -35363832363365653137313830356666323631653535363063366134303164303364643366613466 -66613030383131633735313065383736316635613437393439393437613363386234356535353534 -32613436346439643835636665656537336534643665323961666632613462633839616637393033 -62383966616666613565653732616164623136323861313365333532393931353338666565336430 -30626638363234623662333638633962313633306536323964343731653362313935363932626331 -38613935643861353830373839393465356334356361656335303732336363306263356463396665 -62633432613838363431653338356639383138343431303632666361313863663938643134626361 -66363666633431343831396637353463366366306135646163633563376437663663353033333865 -38323638333664623037386531323063376533623962366530636137363065616361646464663431 -32326363343163303535643865356539396664393233633639633639346137663635366133636437 -34373839623932626130336265623565313233646562653063343739333539306631353863393337 -61353531303339343963306463613339386530366432646336333039336639326165386562333835 -38373665396138616264646434316333363838333765353436393433656330373738333662373033 -37633234343265373133336331393538303965326366663134353936613439656561356438306231 -62313338383439653835316566663039303330306564626634346165386366393135646137393835 -63316431396439653738656239643565366632616133653039313736333034353032313533363662 -36383661663331633961353839386464656134623337353561613135303865363463346638346263 -32333634343162393266353235326635336332623566353838353566323030663732643531663234 -65616234643931346533333365643263323135366236316363636566613032643636623130306230 -33666266373432303132303438656533653331333636396235623164353730343961376361383838 -35386330616439363662396165643631616638656466313933396339303632323232363638326263 -62326337373534616463633062613261636631656464336663313239636432373230383434363937 -39353764623636343534613639353032643632373334623536373665383538623135316238393937 -33633666636531306666306432666462306137653036643866653365373265646437663362313835 -65613431643030313833333839666663366435613065323033666234653530663837363462383138 -37363739386231646234386265343531303163613563303833623534353832396131336235633362 -32666631653933376130616632333031306331323931336566663764373262376637313465383535 -35356332663931313862376365346361613632306166643433386436666435333233623361323934 -35373031663863343239636435646165323763313137626161623238383365323662623934383162 -34356236386339316533373139663562393531333831666136626162616366633461333863613063 -34376534663135343939373439303966633133353632616530626433303463656634306539653139 -63376561363331306266343033336162663334663638323231323532373239393562383238383831 -32323635356534636637303534393832326566306562633762313363363563643764663063656131 -38323061363438366330613239643937616430313865356366343435343736346633393736333131 -66656635336234663938373433646465383061366139313230663539303365616261336338363565 -36316130303836363366303437373762353061363733343861613530666663363738363032323631 -37613634656532626436343639383431353532316435376232663962636238383862353830313634 -39376262353332663438313062316430383366393265323064613038363763333465373561663430 -62306638623165386636316531396535313134396232343332336663333834636532393530376334 -34333037323830646330303562353439383031356165376564303338323630383337616131636133 -64663665306466633538323330333731386233306130366634646637663839323130633065313462 -66633337346264313561663135323034633636646131303234363132633335313966646534646461 -62353131346666323862323461623437363930333064373439333166656430646237393832373462 -63393430646531323835386462326635313861646335323564643630376131626137616231303335 -39623533383939353966356636656337343336336130643333323236303739393263633562633534 -38666632316432663332626437626337376533393139353731383835376131643035313866386265 -33323333613766333932333637353563366462303763613562333963373537343837653535653631 -33383331396363396439646534323832613165393062623332393537393731373234396634363237 -36343839363163356232333936663230336266376432373433393964313436616637633865636435 -33633835353337613233616438396537393166346464313563643536623835613636373436633562 -33366633326130646138373539656336363034346438326361333539626563613734636138376533 -62313863613237316237346330326330623462346539356330333265383163353733613464326238 -36363131613461643235373937326130383063353534393133636339373439656238333232653263 -6137343166303139343338663639306434656262343139653437 +37373533663231633738353864666365663334666134346462306537373463616438663334613764 +6264643666356130396536323235333837316431373536370a623833383939373038366263393161 +62623563386130666637353164363331373061643563313861643239313363323234303738343738 +3864363533393962650a633633613138313066336238336363306161306430626165343039356664 +64643333633938306138636334656631316232626539336466313033373430623263376635626238 +33343839656639366363313535663561666535376461616366376232303034343064626334626432 +36663362386561616537343835373839363632363730386666653066343731643330623266353434 +39306664363561316138363862356663366365643738336536353137396463623630343430346137 +63336164393939666266306536363435663661663939333238373639346431636635323834356437 +31316632333731343362303331353338333830356635643431316165633963356264376134623665 +37626263336339386234376534366463646537623631303462366466633034343932346132363532 +65313237303966333864373664303931653833323036316434666338353430303333386338356262 +62633533613061653262323630366230303561623661303233316131373066343666303635323637 +63313739623230336461613530623732636262393239306663643835396365636166616536316336 +39376163633937353564343766326462663362396239383835363030393539393936383535353561 +64643231346465393261626266666532636265356531356463393839636265613431383335663263 +32353864333335333632653461366338386664373939623234306466323933316161633364353437 +31626164386430656432363861316464313336613466383562663838393738633337303139386264 +61363933653237653237353836313733666266666534333331376633643938323264623664663733 +38616565343830616638623032646533653433663935363563336665623638313637643337646439 +63666661366463313131363433643461626361303130653761333538303739653265646263626263 +37373137373038643365326566383237393236613831393833326630386237316133333935336234 +32353732633334393830306435313435363538626566386135326338363630666561623633323865 +62313732363562643766386462646531383238363962313437373735313262633162643666303136 +30393332663865313064653233653761333364306134646163373365376532393838666262366535 +38633135363932636533333139386136363964316538316435343633666637306139313033666531 +39396138346461333238623362343162663366333738616663353963653061663531343230326265 +62353466393564666462633462303939346532323931306533323132643862653265346137353738 +63366238323130656430313831326330623961383237346436633165366634356466636162646261 +30303962386532353739643464653139386336393737316266653764373738373530323863326233 +36383135366336333831383438323261636665666331663338613266666338623630363533366338 +62626231306135643830336235353338396336646238396633363561326163623132653638306131 +39356634313130373862356263393364623337363136626361396536616631326233343730643864 +63633633343662343833333064356432373235653864303863656163643830333339656265646639 +65623636333630643236316434326264356435393837626330383962336161633262356638393266 +66323337346434333563323032393563663831653437336634613665306331366662623530393263 +34313033303966633235323339356461356630636238313031633834383434343035353438383035 +32386462323232623861626530373663333564333932616563613432383134393661303264303130 +30353431353062373533303237653966373762346264613734643730663932643633353631666565 +33633337633561343733623434653862383364646434663333366334323739653933313537363366 +65396532346439653334653739376562343136346266333739396630373166613763313563396133 +30303530326465653339343035623333646238393034323862623765633139396364346339326233 +36656664303536623834326662666537343537643336366138383935343065313538613363373966 +35393961356531623262363163343034656130383235373937643639613331356333323964633966 +65343632303237653939393438636137316332313931353934313034326366623832353363636564 +66363437616663356431613063613535666336383730333337623439653130653439663830633534 +39356266346231323535646265646439326137346361656632323564313435626165643231653735 +62636530383961343662393232666664363034656534663762393262356163336337373862316331 +62346535636464633363663330323335386230633465613965613630613135366131313138653061 +36613930653430643236323162643864353863323264323731333037306433376236306138303032 +65333963333364326665616333353031323761653764316362336337386230633736363766333662 +64316662343265623064643031326534346639353566376661343462613537373862303233316538 +64323364366430326264366232666263396235363736366437346438656265386464323264653733 +63363263636233353761356331613330353535346336366437613837343061663362663031333964 +34396366313363383863303035633934376432393334663364313866396166623563646530663632 +64613336653635636564373632646337386438333431326264373766306564326430326237363964 +65386631393135313566656435353039363638393535613063393730316634313837623630323161 +36356331323165613537386164343831386633663936313537306639633262633239353830313530 +31363066316332373035656134333937373263633561616431323334336335656131383436626464 +64356634626636356331383962643735303335363863343561373736363134646536393431346432 +62373434323665343533636237363461303335663632326332633535303139393433623032613465 +66383137613531376333323431356566653462616330656532633361623338343637613239303433 +62626537373165666562646561373731383436353436323434653562396334313639616661643935 +30616431313936626637353966366531316463616366306335656664613366303665346263633830 +31303939623631636463366132613866343032633365326433356463643832353230653339646161 +34636637373464386162623961373933363863346637313535643134613265383166393634656131 +64343930643162613362386465306465333235663461303566366266303664316565396563333433 +61356130333833653330666633346663373634396634383030346138313938636134303666623962 +62356636633062646639323963653565386464373736626161633565303335336262383230363732 +39393539333262326234333537356461663033303933383237346235373734656536303862333236 +65643563343932333632386635346438323339343463323338383865343362333430346338633932 +32386139343866633139313533663832356633633663616261386364373065373365386334383932 +63663334616532316133353565643936643035346464643936643862333466363166333066326338 +30326462333463393834303836653437353033313736643365313739313334383030636535656434 +32373231366462643233363837373133366139366130393537363361346133653839396432343936 +34303030373638303934343463326365663466633963323033323066383734336331353864313564 +64393638353735646233326638346464336637373465663562336661363834343138376635323363 +65356537356137336563383866356665383132366136633035646363393933643165656236313863 +30386363393466636435643666663533623332656232393537656161613837663064353838653835 +66666164386566663536653833613062333432363966303066323563363664333464633038353561 +32626634643264373132323638643766386362613639303262633462663737663337393631303436 +62326438396235666338313033383835306566366365396331653633323335333865366565343036 +62313835333964366537323565653662323563383663323033623939346533656665303235313264 +32623163383530376233353738633062346530376161353664333166323538666465363237616166 +34633561636234623263663535333939306239303334363939616636343334383838643939636664 +63623331306332386634646533396232323063336166613339636533353337306531636135313465 +39636164323537643230353234303666313161633366633831356530633635636561373461643136 +66323734386239656635633365373734643038636566626666646434663061333530633837633166 +38663633393764346565316639323638623039326332646432666232656463626335313737623862 +66386439616635633061356461343736656138633064653733643737636361626366313733356665 +34346565613330663938333563623436323934353934393235356234613766633235316137376632 +32386333373235393837393035653538303433373733386562623037363837396431393531303839 +31633162386637333963633335383662623661336133653866653336376366363665623165633032 +65636261616664326661333930343862663766393035613164313837333561326330333930643166 +38393161316364306139376234643363346564613463613536613534373936653532353861633139 +39663130316130366333303834386162343336346664316439373839333465356564303062633434 +62626337616366303730663661383863653834623537333933616239386264326634656235363166 +63323061343639666135306437356665313466326339356230376234363964653631643561633236 +31386339343033353062383231666135303338356335366335663763383237626138633961393239 +66363539333161313138626332303662366336323266643062663762323561656634636539336265 +34653736306231356135626435653736366133386538643338346463653231653333306633666532 +61323266303764653734643262363564656233653135623234663232353762653766313565613736 +31613432363333366233323165616561646366643634376639343966373635343335343661636462 +32613563613139363932636330663764353164316231333461653965373635363337633662653962 +35346463643363376633356234626464666332646331643132643138346665393937 diff --git a/ansible/inventory/group_vars/ops_servers/forgejo.yaml b/ansible/inventory/group_vars/ops_servers/forgejo.yaml index 8c2bd4c..c8ef54d 100644 --- a/ansible/inventory/group_vars/ops_servers/forgejo.yaml +++ b/ansible/inventory/group_vars/ops_servers/forgejo.yaml @@ -1,25 +1,25 @@ --- -# Forgejo Git server configuration +forgejo_version: "15.0.11" -# Forgejo version and domain -forgejo_version: "12" +# The instance answers on every name below. forgejo_domain is the +# canonical one (ROOT_URL, clone URLs, mail links); the rest are aliases. forgejo_domain: git.asxp.io +forgejo_alias_domains: + - git.czsk.it -# Forgejo installation path forgejo_base_path: "{{ services_base_path }}/forgejo" - -# Forgejo ports forgejo_http_port: 3000 -# Forgejo secrets (stored in encrypted vault) +# app.ini is owned by the container's git user +forgejo_uid: 1000 +forgejo_gid: 1000 + +traefik_network: traefik + forgejo_secret_key: "{{ vault_forgejo_secret_key }}" forgejo_internal_token: "{{ vault_forgejo_internal_token }}" forgejo_jwt_secret: "{{ vault_forgejo_jwt_secret }}" forgejo_lfs_jwt_secret: "{{ vault_forgejo_lfs_jwt_secret }}" - -# Backup configuration -backup_dest_dir: /opt/forgejo-backup -backup_old_server: asxpio-old - -# Docker network for Traefik integration -traefik_network: traefik +forgejo_server_lfs_jwt_secret: "{{ vault_forgejo_server_lfs_jwt_secret }}" +forgejo_smtp_user: "{{ vault_forgejo_smtp_user }}" +forgejo_smtp_password: "{{ vault_forgejo_smtp_password }}" diff --git a/ansible/inventory/group_vars/ops_servers/traefik.yaml b/ansible/inventory/group_vars/ops_servers/traefik.yaml index 45f6340..ce8f47c 100644 --- a/ansible/inventory/group_vars/ops_servers/traefik.yaml +++ b/ansible/inventory/group_vars/ops_servers/traefik.yaml @@ -1,11 +1,7 @@ --- -# Traefik reverse proxy configuration - -# Traefik installation path +traefik_version: "v3.7.18" traefik_base_path: "{{ services_base_path }}/traefik" - -# Traefik ports traefik_dashboard_port: 8080 +traefik_network: traefik -# SSL/TLS configuration letsencrypt_email: "{{ vault_letsencrypt_email }}" diff --git a/ansible/inventory/group_vars/services_servers/forgejo_runner.yaml b/ansible/inventory/group_vars/services_servers/forgejo_runner.yaml index a4d04f3..93b47c4 100644 --- a/ansible/inventory/group_vars/services_servers/forgejo_runner.yaml +++ b/ansible/inventory/group_vars/services_servers/forgejo_runner.yaml @@ -15,3 +15,6 @@ forgejo_runner_token: "{{ vault_forgejo_runner_token }}" # Runner installation path forgejo_runner_base_path: "{{ services_base_path }}/forgejo-runner" + +# The runner talks to Forgejo over the VPC, not through Traefik +forgejo_http_port: 3000 diff --git a/ansible/inventory/group_vars/services_servers/minecraft.yaml b/ansible/inventory/group_vars/services_servers/minecraft.yaml new file mode 100644 index 0000000..6914552 --- /dev/null +++ b/ansible/inventory/group_vars/services_servers/minecraft.yaml @@ -0,0 +1,4 @@ +--- +minecraft_fossils_owner: asxpi +minecraft_fossils_rcon_password: "{{ vault_minecraft_fossils_rcon_password }}" +minecraft_fossils_cf_api_key: "{{ vault_minecraft_fossils_cf_api_key }}" diff --git a/ansible/inventory/host_vars/asxpio-ops.yaml b/ansible/inventory/host_vars/asxpio-ops.yaml index 2ec30e1..18a613a 100644 --- a/ansible/inventory/host_vars/asxpio-ops.yaml +++ b/ansible/inventory/host_vars/asxpio-ops.yaml @@ -1,12 +1,12 @@ --- -# Host-specific configuration for asxpio-ops -# Operations server: Forgejo, Traefik, WireGuard VPN hub +public_ip: 135.181.27.77 +vpc_ip: "{{ asxpio_ops_vpc_ip }}" +wireguard_ip: "{{ asxpio_ops_wireguard_ip }}" -# Allow public SSH access for Forgejo git operations +# SSH: the git user may connect from anywhere (Forgejo passthrough) ssh_allow_public_git: true ssh_git_user: git -# Off: the cache reserve never fit the 38 GB disk, so the mounts failed at -# every boot. Backups go to Hetzner S3 with rclone instead. +# s3fs is off here: its ensure_diskfree reserve never fit the 38 GB disk s3fs_enabled: false -s3fs_ensure_diskfree: 20000 # Total:40GB Keep:20GB free = ~15GB for Cache +s3fs_ensure_diskfree: 20000 diff --git a/ansible/inventory/host_vars/asxpio-services.yaml b/ansible/inventory/host_vars/asxpio-services.yaml index e555e4c..02be751 100644 --- a/ansible/inventory/host_vars/asxpio-services.yaml +++ b/ansible/inventory/host_vars/asxpio-services.yaml @@ -1,9 +1,8 @@ --- -# Host-specific configuration for asxpio-services -# Services server: Forgejo Runner, Immich, game servers, XMPP +public_ip: 46.62.228.216 +vpc_ip: "{{ asxpio_services_vpc_ip }}" +wireguard_ip: "{{ asxpio_services_wireguard_ip }}" -# Restrict SSH to VPC and WireGuard networks only ssh_allow_public_git: false - -s3fs_ensure_diskfree: 81920 # 80GB free = max 80GB cache +s3fs_ensure_diskfree: 81920 # 80 GB free = max 80 GB cache diff --git a/ansible/inventory/hosts.yaml b/ansible/inventory/hosts.yaml index 6b4907d..685ce17 100644 --- a/ansible/inventory/hosts.yaml +++ b/ansible/inventory/hosts.yaml @@ -1,6 +1,6 @@ --- -# Ansible inventory for asxpio infrastructure -# Update IP addresses after running terraform apply +# Default inventory: run from a workstation over WireGuard, as root. +# CI uses inventory/ci.yaml instead (VPC addresses, user deploy). all: vars: @@ -8,59 +8,13 @@ all: ansible_python_interpreter: /usr/bin/python3 children: - # Production asxpio infrastructure (excludes legacy servers) asxpio: - vars: - # Common variables for all asxpio production servers - timezone: UTC - locale: en_US.UTF-8 - children: ops_servers: hosts: asxpio-ops: - # Stage 1: Use public IP for WireGuard deployment - # ansible_host: 135.181.27.77 # Uncomment for Stage 1 if needed - # Stage 2: Use WireGuard IP (uncomment below after 01-wireguard-setup.yaml) - ansible_host: "{{ asxpio_ops_wireguard_ip }}" # 10.20.0.10 via WireGuard - public_ip: 135.181.27.77 - vpc_ip: "{{ asxpio_ops_vpc_ip }}" - wireguard_ip: "{{ asxpio_ops_wireguard_ip }}" - server_role: ops - server_type: cx22 - services: - - traefik - - forgejo - - grafana - - prometheus - - loki - - static_sites - + ansible_host: "{{ wireguard_ip }}" services_servers: hosts: asxpio-services: - # Stage 1: Use public IP for WireGuard deployment - # ansible_host: 46.62.228.216 # Uncomment for Stage 1 if needed - # Stage 2: Use WireGuard IP (uncomment below after 01-wireguard-setup.yaml) - ansible_host: "{{ asxpio_services_wireguard_ip }}" # 10.20.0.20 via WireGuard - public_ip: 46.62.228.216 - vpc_ip: "{{ asxpio_services_vpc_ip }}" - wireguard_ip: "{{ asxpio_services_wireguard_ip }}" - server_role: services - server_type: cx43 - services: - - forgejo_runner - - immich - - minecraft - - valheim - - factorio - - prosody - - coturn - - # Legacy servers (standalone, not part of asxpio group) - old_servers: - hosts: - asxpio-old: - ansible_host: asxp.io - server_role: legacy - ansible_user: asxpi + ansible_host: "{{ wireguard_ip }}" diff --git a/ansible/playbooks/02-full-deployment.yaml b/ansible/playbooks/02-full-deployment.yaml deleted file mode 100644 index 1d506b2..0000000 --- a/ansible/playbooks/02-full-deployment.yaml +++ /dev/null @@ -1,256 +0,0 @@ ---- -# Stage 2: Full Infrastructure Deployment -# -# PREREQUISITES: -# 1. Stage 1 (01-wireguard-setup.yaml) must be completed -# 2. inventory/hosts.yaml must be updated to use WireGuard IPs (10.20.0.x) -# 3. You must be connected via WireGuard VPN -# -# This playbook deploys the complete infrastructure in one go: -# - SSH hardening (with connection recovery) -# - Traefik reverse proxy -# - Forgejo Git server -# - Forgejo CI/CD runner -# -# Usage: -# ansible-playbook playbooks/02-full-deployment.yaml -# - -- name: Verify prerequisites - hosts: localhost - gather_facts: false - tasks: - - name: Test WireGuard connectivity - ansible.builtin.wait_for: - host: "{{ asxpio_ops_wireguard_ip }}" - port: 22 - timeout: 10 - state: started - register: wg_test - failed_when: false - - - name: Verify WireGuard is reachable - ansible.builtin.assert: - that: - - not wg_test.failed - fail_msg: | - ❌ ERROR: Cannot reach servers via WireGuard! - - Tested: {{ asxpio_ops_wireguard_ip }}:22 - Result: Connection failed - - Stage 2 automatically uses WireGuard IPs and REQUIRES an active VPN connection. - - To fix: - 1. Ensure Stage 1 is complete: - ansible-playbook playbooks/01-wireguard-setup.yaml - - 2. Import WireGuard config: - playbooks/wireguard-configs//.conf - - 3. Start WireGuard: - - Linux/Mac: sudo wg-quick up .conf - - Windows: Import in WireGuard GUI and activate - - 4. Test connectivity: - ping {{ asxpio_ops_wireguard_ip }} - ping {{ asxpio_services_wireguard_ip }} - - 5. Re-run this playbook - - Aborting deployment for safety... - success_msg: "✅ WireGuard connectivity verified" - - - name: Display pre-deployment checklist - ansible.builtin.debug: - msg: | - ======================================== - Pre-Deployment Checklist - ======================================== - - ✅ WireGuard connectivity verified - ✅ Ready for deployment - - This playbook will: - 1. Verify common setup - 2. Harden SSH (will restrict access to WireGuard/VPC only) - 3. Deploy Traefik reverse proxy - 4. Deploy Forgejo Git server - 5. Deploy Forgejo CI/CD runner (requires manual token setup) - - ⚠️ IMPORTANT: This playbook uses WireGuard IPs automatically. - After SSH hardening, you will ONLY be able to - access servers via WireGuard VPN! - - ======================================== - - - name: Pause for confirmation - ansible.builtin.pause: - prompt: "Press ENTER to continue or Ctrl+C to abort" - -- name: Harden SSH configuration - hosts: asxpio - become: true - gather_facts: true - vars: - # Force WireGuard IPs for Stage 2 (SSH hardening requires it) - ansible_host: "{{ wireguard_ip }}" - - pre_tasks: - - name: Display SSH hardening notice - ansible.builtin.debug: - msg: | - ======================================== - SSH Hardening - {{ inventory_hostname }} - ======================================== - - SSH will be restarted asynchronously. - Connection may drop briefly but will automatically recover. - - Trusted networks: - - WireGuard: {{ wireguard_network }} - - VPC: {{ vpc_network }} - ======================================== - - roles: - - role: ssh_hardening - - post_tasks: - - name: Verify SSH connection after hardening - ansible.builtin.debug: - msg: "SSH hardening complete and connection verified on {{ inventory_hostname }}" - -- name: Deploy Traefik reverse proxy - hosts: asxpio-ops - become: true - gather_facts: true - vars: - # Force WireGuard IPs for Stage 2 - ansible_host: "{{ wireguard_ip }}" - - roles: - - role: traefik - - post_tasks: - - name: Display Traefik status - ansible.builtin.debug: - msg: | - Traefik deployed successfully on {{ inventory_hostname }} - Dashboard: http://{{ asxpio_ops_wireguard_ip }}:{{ traefik_dashboard_port }} - -- name: Deploy Forgejo Git server - hosts: asxpio-ops - become: true - gather_facts: true - vars: - # Force WireGuard IPs for Stage 2 - ansible_host: "{{ wireguard_ip }}" - - roles: - - role: forgejo - - post_tasks: - - name: Display Forgejo status - ansible.builtin.debug: - msg: | - Forgejo deployed successfully on {{ inventory_hostname }} - Web UI: http://{{ asxpio_ops_wireguard_ip }}:{{ forgejo_http_port }} - SSH: ssh://git@{{ asxpio_ops_wireguard_ip }} - - ⚠️ IMPORTANT: Before deploying the runner, you must: - 1. Access Forgejo web UI - 2. Complete initial setup (create admin user) - 3. Go to: Site Administration → Actions → Runners - 4. Generate a runner registration token - 5. Update forgejo_runner_token in inventory/group_vars/all.yaml (or vault) - - - name: Pause for Forgejo runner token generation - ansible.builtin.pause: - prompt: | - - ======================================== - Forgejo Runner Registration Token Required - ======================================== - - The runner cannot be deployed without a registration token from Forgejo. - - Steps to generate the token: - 1. Open Forgejo: http://{{ asxpio_ops_wireguard_ip }}:{{ forgejo_http_port }} - 2. Complete initial setup if not done (create admin account) - 3. Login as admin - 4. Navigate to: Site Administration → Actions → Runners - 5. Click "Create new Runner" or "Generate Token" - 6. Copy the registration token - 7. Update forgejo_runner_token in inventory/group_vars/all.yaml - OR better: ansible-vault edit inventory/group_vars/vault.yaml - - Press ENTER when ready to continue with runner deployment, or Ctrl+C to abort - ======================================== - -- name: Deploy Forgejo CI/CD Runner - hosts: asxpio-services - become: true - gather_facts: true - vars: - # Force WireGuard IPs for Stage 2 - ansible_host: "{{ wireguard_ip }}" - - pre_tasks: - - name: Verify runner token is configured - ansible.builtin.assert: - that: - - forgejo_runner_token is defined - - forgejo_runner_token | length > 0 - - forgejo_runner_token != "REPLACE_WITH_ACTUAL_TOKEN" - fail_msg: | - ERROR: forgejo_runner_token is not properly configured! - - You must set a valid runner registration token from Forgejo: - 1. Get token from: Site Administration → Actions → Runners - 2. Set in inventory/group_vars/all.yaml or vault.yaml - 3. Re-run this playbook - success_msg: "Runner token validated" - - roles: - - role: forgejo_runner - - post_tasks: - - name: Display runner status - ansible.builtin.debug: - msg: | - Forgejo Runner deployed successfully on {{ inventory_hostname }} - Check status: docker logs forgejo-runner - -- name: Display deployment summary - hosts: localhost - gather_facts: false - tasks: - - name: Show final status - ansible.builtin.debug: - msg: | - ======================================== - Full Infrastructure Deployment Complete! - ======================================== - - Services deployed: - - Traefik (asxpio-services): - - Dashboard: http://{{ asxpio_services_wireguard_ip }}:8080 - - Forgejo (asxpio-ops): - - Web UI: http://{{ asxpio_ops_wireguard_ip }}:3000 - - Git SSH: ssh://git@{{ asxpio_ops_wireguard_ip }}:2222 - - Forgejo Runner (asxpio-services): - - Status: docker logs forgejo-runner - - SSH Access: - - Restricted to WireGuard (10.20.0.0/24) and VPC (10.10.0.0/20) - - Connect via WireGuard VPN to access servers - - Next steps: - - Configure Forgejo admin account - - Set up repositories - - Test CI/CD pipeline - - ======================================== diff --git a/ansible/playbooks/02-ssh-hardening.yaml b/ansible/playbooks/02-ssh-hardening.yaml new file mode 100644 index 0000000..ddfecb4 --- /dev/null +++ b/ansible/playbooks/02-ssh-hardening.yaml @@ -0,0 +1,15 @@ +--- +# Restricts SSH to the WireGuard and VPC networks (plus the public git +# user on asxpio-ops). Run by hand over WireGuard: a mistake here locks +# everyone out, so it is deliberately not part of site.yaml. +# +# ansible-playbook playbooks/02-ssh-hardening.yaml --check --diff + +- name: Harden SSH configuration + hosts: asxpio + become: true + gather_facts: true + vars: + ansible_host: "{{ wireguard_ip }}" + roles: + - role: ssh_hardening diff --git a/ansible/playbooks/deploy-forgejo-runner-standalone.yaml b/ansible/playbooks/deploy-forgejo-runner-standalone.yaml deleted file mode 100644 index 8ae3faa..0000000 --- a/ansible/playbooks/deploy-forgejo-runner-standalone.yaml +++ /dev/null @@ -1,226 +0,0 @@ ---- -# Standalone Forgejo Runner Deployment -# -# PREREQUISITES: -# 1. Forgejo must be deployed and running -# 2. You must have created an admin account in Forgejo -# 3. You must generate a runner registration token from Forgejo UI -# -# To get the registration token: -# 1. Login to Forgejo as admin -# 2. Go to: Site Administration → Actions → Runners -# 3. Click "Create new Runner" or "Generate Token" -# 4. Copy the token and update forgejo_runner_token variable -# -# Usage: -# ansible-playbook playbooks/deploy-forgejo-runner-standalone.yaml -# - -- name: Verify Forgejo is accessible - hosts: localhost - gather_facts: false - tasks: - - name: Check Forgejo is running - ansible.builtin.uri: - url: "http://{{ asxpio_ops_vpc_ip }}:{{ forgejo_http_port }}/" - status_code: 200 - timeout: 10 - register: forgejo_check - failed_when: false - delegate_to: localhost - - - name: Display Forgejo status - ansible.builtin.debug: - msg: | - {% if forgejo_check.status == 200 %} - ✅ Forgejo is accessible at http://{{ asxpio_ops_vpc_ip }}:{{ forgejo_http_port }} - {% else %} - ❌ ERROR: Cannot reach Forgejo! - - Please ensure: - 1. Forgejo is deployed and running - 2. You can access: http://{{ asxpio_ops_vpc_ip }}:{{ forgejo_http_port }} - 3. You are connected to the VPC network or WireGuard VPN - - Deploy Forgejo first: - ansible-playbook playbooks/02-full-deployment.yaml - - Or deploy just Forgejo: - ansible-playbook playbooks/legacy/deploy-forgejo.yaml - {% endif %} - - - name: Fail if Forgejo is not accessible - ansible.builtin.fail: - msg: "Forgejo is not accessible. Please deploy Forgejo first." - when: forgejo_check.status != 200 - -- name: Get runner registration token - hosts: localhost - gather_facts: false - tasks: - - name: Prompt for runner token configuration - ansible.builtin.pause: - prompt: | - - ======================================== - Forgejo Runner Registration Token Required - ======================================== - - Before deploying the runner, you MUST generate a registration token from Forgejo. - - Steps to generate the token: - - 1. Open Forgejo: http://{{ asxpio_ops_vpc_ip }}:{{ forgejo_http_port }} - - 2. Login as administrator - - 3. Navigate to: Site Administration → Actions → Runners - (Click your profile icon → Site Administration → Actions → Runners) - - 4. Click "Create new Runner" or "Generate registration token" - - 5. Copy the generated token (it looks like: "eyJhbGc...") - - 6. Update the token in one of these locations: - - Option A - Plaintext (NOT RECOMMENDED): - Edit: inventory/group_vars/all.yaml - Set: forgejo_runner_token: "YOUR-TOKEN-HERE" - - Option B - Ansible Vault (RECOMMENDED): - ansible-vault edit inventory/group_vars/vault.yaml - Set: vault_forgejo_runner_token: "YOUR-TOKEN-HERE" - Then in all.yaml: forgejo_runner_token: "{{ vault_forgejo_runner_token }}" - - 7. Save the file - - 8. Press ENTER to continue with runner deployment - - Or press Ctrl+C to abort and configure the token first - ======================================== - -- name: Deploy Forgejo CI/CD Runner - hosts: asxpio-services - become: true - gather_facts: true - - pre_tasks: - - name: Verify runner token is configured - ansible.builtin.assert: - that: - - forgejo_runner_token is defined - - forgejo_runner_token | length > 0 - - forgejo_runner_token != "REPLACE_WITH_ACTUAL_TOKEN" - - forgejo_runner_token != "***REDACTED***" # Old token from git - fail_msg: | - ❌ ERROR: forgejo_runner_token is not properly configured! - - Current value: {{ forgejo_runner_token | default('NOT SET') }} - - You must set a valid runner registration token from Forgejo: - - 1. Get token from Forgejo UI: - Site Administration → Actions → Runners → Generate Token - - 2. Update in inventory/group_vars/all.yaml: - forgejo_runner_token: "YOUR-TOKEN-HERE" - - OR better, use Ansible Vault (recommended): - ansible-vault edit inventory/group_vars/vault.yaml - vault_forgejo_runner_token: "YOUR-TOKEN-HERE" - - 3. Re-run this playbook - - Aborting deployment... - success_msg: "✅ Runner token validated (length: {{ forgejo_runner_token | length }} chars)" - - - name: Display deployment information - ansible.builtin.debug: - msg: | - ======================================== - Deploying Forgejo Runner - ======================================== - - Target: {{ inventory_hostname }} - Forgejo instance: http://{{ asxpio_ops_vpc_ip }}:{{ forgejo_http_port }} - Runner name: runner-{{ inventory_hostname }} - - This will: - 1. Deploy runner container - 2. Register with Forgejo instance - 3. Start accepting CI/CD jobs - ======================================== - - roles: - - role: forgejo_runner - - post_tasks: - - name: Wait for runner registration - ansible.builtin.pause: - seconds: 5 - prompt: "Waiting for runner to register with Forgejo..." - - - name: Display runner status - ansible.builtin.debug: - msg: | - ======================================== - Forgejo Runner Deployment Complete! - ======================================== - - Runner deployed successfully on {{ inventory_hostname }} - - Check status: - docker logs forgejo-runner - - Verify in Forgejo UI: - 1. Login to Forgejo - 2. Go to: Site Administration → Actions → Runners - 3. You should see: runner-{{ inventory_hostname }} (idle/active) - - Test the runner: - 1. Create a repository with .forgejo/workflows/test.yml - 2. Push a commit - 3. Check Actions tab for workflow run - - Runner supports these labels: - - ubuntu-latest, ubuntu-22.04, ubuntu-24.04 - - debian-latest, debian-12, debian-11 - - node-20, node-22 - - python-3.11, python-3.12 - - alpine-latest, arch-latest - - nginx-latest, nginx-alpine - - Example workflow: - .forgejo/workflows/test.yml: - --- - name: Test - on: [push] - jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - run: echo "Hello from Forgejo Actions!" - - ======================================== - -- name: Verify runner in Forgejo - hosts: localhost - gather_facts: false - tasks: - - name: Reminder to check Forgejo UI - ansible.builtin.debug: - msg: | - - ⚠️ FINAL STEP: Verify runner in Forgejo UI - - 1. Open: http://{{ asxpio_ops_vpc_ip }}:{{ forgejo_http_port }} - 2. Login as admin - 3. Go to: Site Administration → Actions → Runners - 4. Verify runner-asxpio-services appears in the list - 5. Status should be "idle" (green) or "active" - - If the runner doesn't appear: - - Check logs: docker logs forgejo-runner - - Verify token is correct - - Ensure runner can reach Forgejo at {{ asxpio_ops_vpc_ip }}:{{ forgejo_http_port }} diff --git a/ansible/playbooks/legacy/backup-forgejo.yaml b/ansible/playbooks/legacy/backup-forgejo.yaml deleted file mode 100644 index 4e0cb93..0000000 --- a/ansible/playbooks/legacy/backup-forgejo.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: Backup Forgejo from old server - hosts: asxpio-old - become: true - roles: - - forgejo_backup diff --git a/ansible/playbooks/legacy/restore-forgejo.yaml b/ansible/playbooks/legacy/restore-forgejo.yaml deleted file mode 100644 index 5f32859..0000000 --- a/ansible/playbooks/legacy/restore-forgejo.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: Restore Forgejo to new server - hosts: asxpio-ops - become: true - roles: - - forgejo_restore diff --git a/ansible/roles/base/defaults/main.yaml b/ansible/roles/base/defaults/main.yaml new file mode 100644 index 0000000..1ff96d5 --- /dev/null +++ b/ansible/roles/base/defaults/main.yaml @@ -0,0 +1,4 @@ +--- +journald_max_use: 500M +# Weekly cleanup of images no container uses; every deploy leaves one behind +docker_prune_schedule: "0 3 * * 1" diff --git a/ansible/roles/base/files/20auto-upgrades b/ansible/roles/base/files/20auto-upgrades new file mode 100644 index 0000000..8d6d7c8 --- /dev/null +++ b/ansible/roles/base/files/20auto-upgrades @@ -0,0 +1,2 @@ +APT::Periodic::Update-Package-Lists "1"; +APT::Periodic::Unattended-Upgrade "1"; diff --git a/ansible/roles/base/files/daemon.json b/ansible/roles/base/files/daemon.json new file mode 100644 index 0000000..a611855 --- /dev/null +++ b/ansible/roles/base/files/daemon.json @@ -0,0 +1,7 @@ +{ + "live-restore": true, + "storage-driver": "overlay2", + "log-opts": { + "max-size": "20m" + } +} diff --git a/ansible/roles/base/handlers/main.yaml b/ansible/roles/base/handlers/main.yaml new file mode 100644 index 0000000..5700f82 --- /dev/null +++ b/ansible/roles/base/handlers/main.yaml @@ -0,0 +1,12 @@ +--- +- name: Restart journald + ansible.builtin.systemd: + name: systemd-journald + state: restarted + +# live-restore is on, so containers keep running across this restart +- name: Restart docker + ansible.builtin.systemd: + name: docker + state: restarted + daemon_reload: true diff --git a/ansible/roles/base/tasks/main.yaml b/ansible/roles/base/tasks/main.yaml new file mode 100644 index 0000000..0c9b9d0 --- /dev/null +++ b/ansible/roles/base/tasks/main.yaml @@ -0,0 +1,76 @@ +--- +# Host baseline shared by both servers. Users, Docker itself and the +# first-boot packages come from terraform/cloud-init; SSH and WireGuard +# have their own bootstrap playbooks. + +- name: Install baseline packages + ansible.builtin.apt: + name: + - unattended-upgrades + - fail2ban + state: present + +- name: Enable unattended security upgrades + ansible.builtin.copy: + src: 20auto-upgrades + dest: /etc/apt/apt.conf.d/20auto-upgrades + mode: "0644" + +- name: Keep fail2ban running (sshd jail from the Debian defaults) + ansible.builtin.systemd: + name: fail2ban + enabled: true + state: started + +- name: Create journald drop-in directory + ansible.builtin.file: + path: /etc/systemd/journald.conf.d + state: directory + mode: "0755" + +- name: Cap the journal size + ansible.builtin.copy: + dest: /etc/systemd/journald.conf.d/size.conf + content: | + [Journal] + SystemMaxUse={{ journald_max_use }} + mode: "0644" + notify: Restart journald + +- name: Configure the Docker daemon + ansible.builtin.copy: + src: daemon.json + dest: /etc/docker/daemon.json + mode: "0644" + notify: Restart docker + +# dockerd must only listen on its unix socket. A drop-in once added +# -H tcp://:2375, which gave every container on the private +# network unauthenticated root on this host. +- name: Remove any Docker systemd override + ansible.builtin.file: + path: /etc/systemd/system/docker.service.d/override.conf + state: absent + notify: Restart docker + +- name: Prune unused Docker images weekly + ansible.builtin.copy: + dest: /etc/cron.d/docker-image-prune + content: | + {{ docker_prune_schedule }} root docker image prune -af > /var/log/docker-prune.log 2>&1 + mode: "0644" + +- name: Check for the old prune entry in root's crontab + ansible.builtin.command: crontab -l + register: root_crontab + changed_when: false + failed_when: false + check_mode: false + +- name: Drop the old prune entry from root's crontab + ansible.builtin.shell: | + set -o pipefail + crontab -l | grep -v 'docker image prune' | crontab - + args: + executable: /bin/bash + when: "'docker image prune' in root_crontab.stdout" diff --git a/ansible/roles/crawler_alert/defaults/main.yaml b/ansible/roles/crawler_alert/defaults/main.yaml new file mode 100644 index 0000000..72040b6 --- /dev/null +++ b/ansible/roles/crawler_alert/defaults/main.yaml @@ -0,0 +1,8 @@ +--- +# Daily/weekly AI-crawler report for the Forgejo router, mailed through +# Forgejo's own SMTP settings. Source and docs: +# https://git.asxp.io/asxpi/traefik-crawlers-statistics +crawler_recipient: "{{ vault_crawler_recipient }}" +crawler_site: git.asxp.io +crawler_threshold: 100000 +crawler_robots_url: https://git.czsk.it/robots.txt diff --git a/ansible/roles/crawler_alert/files/crawler-alert.py b/ansible/roles/crawler_alert/files/crawler-alert.py new file mode 100644 index 0000000..fa6e84b --- /dev/null +++ b/ansible/roles/crawler_alert/files/crawler-alert.py @@ -0,0 +1,707 @@ +#!/usr/bin/env python3 +"""Crawler watch for a Traefik-fronted Forgejo (or any Traefik site). + +Separates real AI crawlers from traffic that only claims to be, by +checking the source IP against each vendor's published range list. + +Modes: + (default) daily threshold alert: mail only if yesterday exceeded + CRAWLER_THRESHOLD requests + --weekly summary: always mail stats for the past 7 days + --dry-run print instead of mailing (combinable with --weekly) + +Configuration is by environment variable; see README.md. The only +value with no sensible default is CRAWLER_RECIPIENT. +""" +import configparser, glob, gzip, ipaddress, json, os, re, smtplib, socket, subprocess, sys, urllib.request +from collections import Counter +from datetime import date, timedelta +from email.mime.text import MIMEText + +# --- configuration ------------------------------------------------------ +THRESHOLD = int(os.environ.get("CRAWLER_THRESHOLD", "100000")) +RECIPIENT = os.environ.get("CRAWLER_RECIPIENT", "") +SITE = os.environ.get("CRAWLER_SITE", "this site") +LOG_GLOB = os.environ.get("CRAWLER_LOG", "/var/log/traefik/access.log*") +# Traefik router name to filter on, e.g. "forgejo@docker". Empty = all +# traffic reaching Traefik, which is what you want for a single-site host. +ROUTER = os.environ.get("CRAWLER_ROUTER", "") +# Container to read SMTP settings from (Forgejo/Gitea app.ini [mailer]). +# Set CRAWLER_SMTP_* instead to configure SMTP directly. +MAIL_CONTAINER = os.environ.get("CRAWLER_MAIL_CONTAINER", "forgejo") +MAIL_INI = os.environ.get("CRAWLER_MAIL_INI", "/data/gitea/conf/app.ini") + +WEEKLY = "--weekly" in sys.argv +DRY = "--dry-run" in sys.argv + +if not RECIPIENT and not DRY: + sys.exit("CRAWLER_RECIPIENT is not set (no --dry-run either); refusing " + "to run. See README.md.") + +# --- verified-crawler support ------------------------------------------- +# Vendors publishing machine-readable IP ranges. A UA token alone proves +# nothing: on the instance this was written for, ~83% of AI-bot-labelled +# traffic came from hosts outside these ranges. Only IP-verified hits count. +RANGE_SOURCES = { + "openai": ["https://openai.com/chatgpt-user.json", + "https://openai.com/gptbot.json", + "https://openai.com/searchbot.json"], + "anthropic": ["https://claude.com/crawling/bots.json"], + "perplexity":["https://www.perplexity.ai/perplexitybot.json"], + "apple": ["https://search.developer.apple.com/applebot.json"], + "google": ["https://developers.google.com/static/crawling/ipranges/special-crawlers.json", + "https://developers.google.com/static/crawling/ipranges/common-crawlers.json", + "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers.json", + "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers-google.json"], +} +# UA token -> vendor whose ranges must contain the source IP +BOT_VENDOR = { + "ChatGPT-User": "openai", "GPTBot": "openai", "OAI-SearchBot": "openai", + "ClaudeBot": "anthropic", "Claude-User": "anthropic", "Claude-SearchBot": "anthropic", + "PerplexityBot": "perplexity", "Perplexity-User": "perplexity", + "Applebot-Extended": "apple", "Applebot": "apple", + "Googlebot": "google", "Google-Extended": "google", "GoogleOther": "google", +} + +# --- reverse-DNS verification ------------------------------------------- +# For vendors that publish no IP ranges. The check is forward-confirmed +# (FCrDNS): PTR of the address must end in one of the vendor's domains, +# AND that hostname must resolve back to the same address. The forward +# step is what makes it meaningful - a PTR record alone is set by +# whoever controls the address block, so it proves nothing on its own. +BOT_RDNS = { + "Amazonbot": (".crawl.amazonbot.amazon",), + "meta-externalagent": (".facebook.com", ".fbsv.net"), + "meta-externalfetcher": (".facebook.com", ".fbsv.net"), + "FacebookBot": (".facebook.com", ".fbsv.net"), + "YouBot": (".search.you.com",), + "Bytespider": (".bytedance.com", ".byteoversea.com"), + "PetalBot": (".petalsearch.com", ".aspiegel.com"), + "DuckAssistBot": (".duckduckgo.com",), + # Apple and Google publish ranges, but also answer rDNS; listed so a + # range miss can still be resolved when the published file is stale. + "Applebot": (".applebot.apple.com",), + "Googlebot": (".googlebot.com", ".google.com"), +} +# rDNS is slow (two lookups per address), so only unique IPs are probed +# and results are memoised for the run. +RDNS_MAX = int(os.environ.get("CRAWLER_RDNS_MAX", "400")) +RDNS_ENABLE = os.environ.get("CRAWLER_RDNS", "1") != "0" + +# --- ASN verification, for single-tenant networks only ------------------- +# An ASN match usually proves nothing: genuine YouBot and genuine +# Amazonbot both live in AS14618, which is also every EC2 instance a +# spoofer could rent. It is only evidence when the ASN belongs to the +# vendor exclusively and sells no compute in it. Meta qualifies - +# AS32934 is Meta's own network and cannot be rented - and Meta +# publishes no IP range file (their docs say to email webmasters@meta.com), +# so without this there is nothing to check them against at all. +BOT_ASN = { + "meta-externalagent": {"32934"}, + "meta-externalfetcher": {"32934"}, + "FacebookBot": {"32934"}, + "facebookexternalhit": {"32934"}, +} +ASN_ENABLE = os.environ.get("CRAWLER_ASN", "1") != "0" +# Team Cymru's DNS interface: .origin[6].asn.cymru.com TXT +CYMRU4 = "origin.asn.cymru.com" +CYMRU6 = "origin6.asn.cymru.com" +# robots.txt to audit verified crawlers against. Fetched once per run; +# empty or unreachable simply skips the compliance check. +ROBOTS_URL = os.environ.get("CRAWLER_ROBOTS", "") +CACHE = os.environ.get("CRAWLER_CACHE", "/var/cache/crawler-alert/ranges.json") +CF_URL = "https://www.cloudflare.com/ips-v4" +# claude.com and cloudflare.com sit behind Cloudflare, which 403s the +# default Python-urllib UA. Send a real one. +UA = "Mozilla/5.0 (compatible; crawler-alert/1.0; +https://codeberg.org/)" + + +def fetch(url, timeout=20): + req = urllib.request.Request(url, headers={"User-Agent": UA}) + with urllib.request.urlopen(req, timeout=timeout) as r: + return r.read() + + +def load_ranges(): + """Fetch vendor prefixes; fall back to cache so a vendor outage + degrades the report instead of breaking the mail.""" + out, stale = {}, [] + for vendor, urls in RANGE_SOURCES.items(): + nets = [] + for u in urls: + try: + for p in json.loads(fetch(u)).get("prefixes", []): + for k, v in p.items(): + if k.startswith("ipv"): + nets.append(v) + except Exception: + stale.append(vendor) + if nets: + out[vendor] = nets + cf = [] + try: + cf = [l.strip() for l in fetch(CF_URL).decode().splitlines() if l.strip()] + except Exception: + stale.append("cloudflare") + try: + cached = json.load(open(CACHE)) + except Exception: + cached = {} + for vendor in RANGE_SOURCES: + if vendor not in out and vendor in cached.get("vendors", {}): + out[vendor] = cached["vendors"][vendor] + if not cf: + cf = cached.get("cloudflare", []) + if out: + try: + os.makedirs(os.path.dirname(CACHE), exist_ok=True) + json.dump({"vendors": out, "cloudflare": cf}, open(CACHE, "w")) + except Exception: + pass + # bucket by first octet so lookups stay cheap over ~1425 prefixes + idx = {} + for vendor, nets in out.items(): + b = {} + for s in nets: + n = ipaddress.ip_network(s) + b.setdefault(int(n.network_address) >> 24 if n.version == 4 else -1, []).append(n) + idx[vendor] = b + cfnets = [ipaddress.ip_network(s) for s in cf] + return idx, cfnets, sorted(set(stale)) + + +def in_index(idx_v, addr): + if idx_v is None: + return False + key = int(addr) >> 24 if addr.version == 4 else -1 + return any(addr in n for n in idx_v.get(key, [])) + + +def load_robots(): + """Disallow patterns from robots.txt, as (prefix, contains) pairs. + + Only the `User-agent: *` group is read: the point is to audit what + well-behaved crawlers were asked not to fetch, not to model every + per-agent rule. Returns [] when unset or unreachable, which disables + the compliance check rather than reporting false violations. + """ + if not ROBOTS_URL: + return [] + try: + txt = fetch(ROBOTS_URL, timeout=15).decode("utf-8", "replace") + except Exception: + return [] + rules, in_star = [], False + for line in txt.splitlines(): + line = line.split("#")[0].strip() + if not line: + continue + k, _, v = line.partition(":") + k, v = k.strip().lower(), v.strip() + if k == "user-agent": + in_star = (v == "*") + elif k == "disallow" and in_star and v: + # Forgejo-style /*/*/thing/ means "contains /thing/". + if v.startswith("/*"): + rules.append(("", v.lstrip("/*"))) + else: + rules.append((v.rstrip("*"), "")) + return rules + + +def robots_blocked(path, rules): + for prefix, contains in rules: + if prefix and path.startswith(prefix): + return True + if contains and contains in path: + return True + return False + + +_asn_cache = {} + + +def asn_of(ip): + """Origin ASN for an address via Team Cymru's DNS interface. + + Returns the ASN as a string, or "" if it cannot be determined. + Needs `dig`; absence degrades to no ASN evidence, never an error. + """ + if ip in _asn_cache: + return _asn_cache[ip] + asn = "" + try: + a = ipaddress.ip_address(ip) + if a.version == 4: + q = ".".join(reversed(ip.split("."))) + "." + CYMRU4 + else: + nibbles = a.exploded.replace(":", "") + q = ".".join(reversed(nibbles)) + "." + CYMRU6 + out = subprocess.run(["dig", "+short", "+time=3", "+tries=1", q, "TXT"], + capture_output=True, text=True, timeout=12).stdout + # "32934 | 2a03:2880:f814::/48 | IE | ripencc | 2011-08-22" + first = out.strip().strip('"').split("|")[0].strip() + if first.split(" ")[0].isdigit(): + asn = first.split(" ")[0] + except (OSError, ValueError, subprocess.SubprocessError): + pass + _asn_cache[ip] = asn + return asn + + +_rdns_cache = {} + + +def fcrdns(ip, suffixes): + """Forward-confirmed reverse DNS. + + PTR must end in one of `suffixes`, and the name it gives must resolve + back to `ip`. Returns (ok, hostname). Without the forward step this + would only prove the address owner can write their own PTR record. + """ + key = (ip, suffixes) + if key in _rdns_cache: + return _rdns_cache[key] + result = (False, "", "mismatch") + try: + host = socket.gethostbyaddr(ip)[0].rstrip(".").lower() + if any(host.endswith(s) or host == s.lstrip(".") for s in suffixes): + _, _, addrs = socket.gethostbyname_ex(host) + try: + v6 = socket.getaddrinfo(host, None, socket.AF_INET6) + addrs = addrs + [a[4][0] for a in v6] + except OSError: + pass + if ip in addrs: + result = (True, host, "ok") + else: + result = (False, host, "mismatch") + else: + result = (False, host, "mismatch") + except OSError: + # No PTR at all. This is absence of evidence, not evidence of + # forgery: Meta's IPv6 crawler space, for one, publishes no PTR + # records. Kept distinct so a real mismatch stays visible. + result = (False, "", "no-ptr") + _rdns_cache[key] = result + return result + +days = [date.today() - timedelta(days=i) for i in range(7 if WEEKLY else 1, 0, -1)] +stamps = [d.strftime("%d/%b/%Y") for d in days] + +RANGES, CFNETS, STALE = (load_ranges() if WEEKLY else ({}, [], [])) + +per_day = {s: [0, set()] for s in stamps} +repos = Counter() +uas = Counter() +verified = Counter() # bot -> IP-verified hits +spoofed = Counter() # bot -> UA claimed, IP outside vendor ranges +proxied = Counter() # bot -> arrived via Cloudflare, unverifiable +ver_paths = Counter() # paths fetched by verified bots +ver_ips = {} # bot -> set of verified source IPs +# What the verified crawlers actually did, once confirmed: status mix, +# bytes served, and any path robots.txt asked them not to fetch. +ver_status = Counter() +ver_bytes = [0] +ver_violations = Counter() # path -> hits, robots.txt disallowed +ver_bot_bytes = Counter() # bot -> bytes served +# Repository *content* fetched by verified crawlers, as opposed to +# policy files and chrome. This is the question a permissive robots.txt +# is asking: now that they may read the code, do they? Counting top +# paths cannot answer it - robots.txt and sitemap.xml drown everything. +ver_content = Counter() # bot -> content hits +ver_content_paths = Counter() +CONTENT_RE = re.compile(r"/(src|raw|commit|commits|media|blame|archive|compare)/") +# Which repositories each vendor is actually interested in. +ver_repos = {} # bot -> Counter(owner/repo) +ver_pkgs = Counter() # "type/name" -> hits, registry artifacts +ver_pkg_index = Counter() # owner -> hits on the bare package index +# Forge paths are /owner/repo/...; these first segments are not owners, +# and a two-segment path like /pix/photo.jpg on another vhost is a file, +# not a repository - hence the extension check. +NOT_OWNER = {"assets", "fonts", "explore", "user", "api", "avatars", + "repo-avatars", "swagger.v1.json", "attachments", "notifications", + "issues", "pulls", "milestones", "admin", "org", "repo", "login", + "-", "favicon.ico", "robots.txt", "sitemap.xml", "pix", "static"} + + +# Container/package registry: /{owner}/-/packages/{type}/{name}[/{version}] +# Worth counting separately from repositories - it says which published +# artifacts crawlers are indexing, not which source they are reading. +PKG_RE = re.compile(r"^/([^/]+)/-/packages(?:/([^/?]+)/([^/?]+))?") + + +def package_of(path): + """(owner, 'type/name') for a registry path, else None. Bare + /owner/-/packages (the index) returns name None.""" + m = PKG_RE.match(path.split("?")[0]) + if not m: + return None + owner, ptype, name = m.group(1), m.group(2), m.group(3) + return (owner, f"{ptype}/{name}" if ptype and name else None) + + +def repo_of(path): + """owner/repo for a forge path, else None.""" + seg = path.lstrip("/").split("?")[0].split("/") + if len(seg) < 2 or not seg[0] or not seg[1]: + return None + if seg[0].lower() in NOT_OWNER or seg[1].lower() in NOT_OWNER: + return None + if "." in seg[1] and len(seg) == 2: # /pix/photo.jpg -> a file + return None + return f"{seg[0]}/{seg[1]}" +# rDNS-verified vendors: {bot: {ip: hits}}, resolved after the log pass +rdns_hits = {b: Counter() for b in BOT_RDNS} +rdns_proxied = Counter() +# {bot: {ip: Counter(path)}} - kept so that once an address is confirmed +# by DNS/ASN its paths can join the verified-path breakdown, which +# otherwise only ever showed IP-range-verified vendors. +rdns_paths = {b: {} for b in BOT_RDNS} +ROBOTS_RULES = load_robots() if WEEKLY else [] + + +def record_verified(bot, path, status, nbytes): + """Log what a confirmed crawler did. Called only after verification, + so these counters describe legitimate traffic and nothing else.""" + ver_paths[path] += 1 + ver_status[status] += 1 + ver_bytes[0] += nbytes + ver_bot_bytes[bot] += nbytes + if CONTENT_RE.search(path): + ver_content[bot] += 1 + ver_content_paths[path] += 1 + pkg = package_of(path) + if pkg: + owner, name = pkg + if name: + ver_pkgs[f"{owner}/{name}"] += 1 + else: + ver_pkg_index[owner] += 1 + r = repo_of(path) + if r: + ver_repos.setdefault(bot, Counter())[r] += 1 + if ROBOTS_RULES and robots_blocked(path, ROBOTS_RULES): + ver_violations[path] += 1 + + +for path in sorted(glob.glob(LOG_GLOB)): + opener = gzip.open if path.endswith(".gz") else open + with opener(path, "rt", errors="replace") as f: + for line in f: + if ROUTER and ROUTER not in line: + continue + for s in stamps: + if s in line: + per_day[s][0] += 1 + per_day[s][1].add(line.split(" ", 1)[0]) + parts = line.split() + if len(parts) > 6 and parts[6].count("/") >= 2: + repos["/".join(parts[6].split("/")[1:3])] += 1 + # Traefik CLF: ... "" "" "" ... + # Anchor on the request counter so this works whatever + # the router is called. + ua = re.search(r"\"[^\"]*\" \"([^\"]*)\" \d+ \"", line) + if ua: + u = ua.group(1) + m = re.search(r"(bot|crawler|spider|scrapy|externalagent|gpt|claude|perplexity)[\w./-]*", u, re.I) + uas[m.group(0) if m else ("(none)" if u == "-" else "(browser-like)")] += 1 + if WEEKLY: + # longest token first so Applebot-Extended wins over Applebot + bot = next((b for b in sorted(BOT_VENDOR, key=len, reverse=True) if b in u), None) + if bot: + try: + addr = ipaddress.ip_address(parts[0]) + except ValueError: + addr = None + if addr is None: + pass + elif any(addr in n for n in CFNETS): + proxied[bot] += 1 + elif in_index(RANGES.get(BOT_VENDOR[bot]), addr): + verified[bot] += 1 + ver_ips.setdefault(bot, set()).add(parts[0]) + if len(parts) > 9: + record_verified( + bot, parts[6][:60], parts[8], + int(parts[9]) if parts[9].isdigit() else 0) + else: + spoofed[bot] += 1 + if WEEKLY and RDNS_ENABLE: + rb = next((b for b in sorted(BOT_RDNS, key=len, reverse=True) + if b in u), None) + # skip if already counted by IP-range check + if rb and not (rb in BOT_VENDOR and rb == bot): + try: + a2 = ipaddress.ip_address(parts[0]) + except ValueError: + a2 = None + if a2 is None: + pass + elif any(a2 in n for n in CFNETS): + rdns_proxied[rb] += 1 + else: + rdns_hits[rb][parts[0]] += 1 + if len(parts) > 9 and not a2.is_private: + rdns_paths[rb].setdefault(parts[0], []).append( + (parts[6][:60], parts[8], + int(parts[9]) if parts[9].isdigit() else 0)) + break + +total = sum(v[0] for v in per_day.values()) +all_ips = set().union(*(v[1] for v in per_day.values())) + +# --- resolve rDNS candidates --------------------------------------------- +# One pass over unique addresses, busiest first, capped by RDNS_MAX so a +# flood of distinct forgeries cannot stall the report on DNS timeouts. +rdns_ok = Counter() # bot -> forward-confirmed by PTR +rdns_mismatch = Counter() # bot -> PTR exists but points elsewhere: forgery +rdns_noptr = Counter() # bot -> no PTR at all: cannot be judged by DNS +rdns_asn = Counter() # bot -> no PTR, but origin ASN is the vendor's +rdns_skipped = Counter() # bot -> hits left unresolved by the cap +rdns_private = Counter() # bot -> hits from RFC1918/loopback (see below) +rdns_ok_ips = {} +if WEEKLY and RDNS_ENABLE: + socket.setdefaulttimeout(3) + for bot, counter in rdns_hits.items(): + budget = RDNS_MAX + for ip, hits in counter.most_common(): + # A private source address means the real client IP never + # reached the log: something in front (Docker's bridge + # gateway, a local proxy) is rewriting it. Neither verified + # nor spoofed - the evidence simply is not there. + if ipaddress.ip_address(ip).is_private: + rdns_private[bot] += hits + continue + if budget <= 0: + rdns_skipped[bot] += hits + continue + budget -= 1 + ok, _host, why = fcrdns(ip, BOT_RDNS[bot]) + confirmed = False + if ok: + rdns_ok[bot] += hits + rdns_ok_ips.setdefault(bot, set()).add(ip) + confirmed = True + elif why == "no-ptr": + # Fall back to ASN, but only for vendors whose network is + # single-tenant (see BOT_ASN). Otherwise leave it unknown. + if ASN_ENABLE and bot in BOT_ASN and asn_of(ip) in BOT_ASN[bot]: + rdns_asn[bot] += hits + rdns_ok_ips.setdefault(bot, set()).add(ip) + confirmed = True + else: + rdns_noptr[bot] += hits + else: + rdns_mismatch[bot] += hits + if confirmed: + for p, st, nb in rdns_paths.get(bot, {}).get(ip, ()): + record_verified(bot, p, st, nb) + socket.setdefaulttimeout(None) + +if not WEEKLY: + s0 = stamps[0] + count, ips = per_day[s0][0], len(per_day[s0][1]) + print(f"{s0}: {count} requests, {ips} unique IPs (threshold {THRESHOLD})") + if DRY or count <= THRESHOLD: + sys.exit(0) + subject = f"{SITE}: {count} requests on {s0} (over {THRESHOLD}/day threshold)" + body = (f"Forgejo served {count} requests from {ips} unique IPs on {s0} - " + f"above the {THRESHOLD}/day watch threshold.\n\n" + "Time to consider Anubis (PoW challenge) in front of Forgejo.\n") +else: + lines = [f"{SITE} weekly crawler report ({stamps[0]} - {stamps[-1]})", ""] + lines.append(f"Total: {total} requests, {len(all_ips)} unique IPs") + lines.append(f"Daily threshold alert fires above {THRESHOLD} req/day (none = quiet week)") + lines.append("") + lines.append("Per day:") + for s in stamps: + lines.append(f" {s}: {per_day[s][0]:>7} requests, {len(per_day[s][1]):>6} unique IPs") + lines.append("") + lines.append("Top repos:") + for r, c in repos.most_common(5): + lines.append(f" {c:>7} {r}") + lines.append("") + lines.append("User-agent classes (browser-like = mostly the spoofing swarm):") + for u, c in uas.most_common(8): + lines.append(f" {c:>7} {u}") + lines.append("") + + # --- legitimate (IP-verified) crawlers ------------------------------- + lines.append("=" * 62) + lines.append("LEGITIMATE CRAWLERS (source IP inside vendor's published ranges)") + lines.append("=" * 62) + if STALE: + lines.append(f"NOTE: range fetch failed for {', '.join(STALE)} - used cache") + tv, ts, tp = sum(verified.values()), sum(spoofed.values()), sum(proxied.values()) + claimed = tv + ts + tp + if not claimed: + lines.append(" No AI-bot user-agents seen this week.") + else: + lines.append(f" {'bot':<20}{'verified':>9}{'spoofed':>9}{'via CF':>8} {'IPs':>4}") + for bot in sorted(set(verified) | set(spoofed) | set(proxied), + key=lambda b: -(verified[b] + spoofed[b] + proxied[b])): + lines.append(f" {bot:<20}{verified[bot]:>9}{spoofed[bot]:>9}" + f"{proxied[bot]:>8} {len(ver_ips.get(bot, ())):>4}") + pct = tv / claimed * 100 + lines.append("") + lines.append(f" {claimed} requests claimed an AI-bot identity; {tv} verified ({pct:.1f}%).") + lines.append(f" {ts} failed IP verification (spoofed). {tp} arrived via Cloudflare") + lines.append(" and cannot be verified by IP - not counted either way.") + # --- vendors verified by reverse DNS --------------------------------- + if RDNS_ENABLE and (rdns_ok or rdns_mismatch or rdns_noptr or rdns_asn + or rdns_proxied or rdns_private): + lines.append("") + lines.append("-" * 62) + lines.append("VERIFIED BY DNS / ASN (vendors publishing no IP ranges)") + lines.append("-" * 62) + lines.append(f" {'bot':<20}{'PTR ok':>7}{'ASN ok':>7}{'FORGED':>8}" + f"{'no-PTR':>7}{'CF':>6}{'priv':>7} {'IPs':>4}") + allbots = (set(rdns_ok) | set(rdns_mismatch) | set(rdns_noptr) + | set(rdns_asn) | set(rdns_proxied) | set(rdns_private)) + for bot in sorted(allbots, key=lambda b: -(rdns_ok[b] + rdns_mismatch[b] + + rdns_noptr[b] + rdns_asn[b] + + rdns_proxied[b] + rdns_private[b])): + lines.append(f" {bot:<20}{rdns_ok[bot]:>7}{rdns_asn[bot]:>7}" + f"{rdns_mismatch[bot]:>8}{rdns_noptr[bot]:>7}" + f"{rdns_proxied[bot]:>6}{rdns_private[bot]:>7}" + f" {len(rdns_ok_ips.get(bot, ())):>4}") + if rdns_skipped: + sk = ", ".join(f"{b} {c}" for b, c in rdns_skipped.most_common(4)) + lines.append(f" (unresolved, over CRAWLER_RDNS_MAX={RDNS_MAX}: {sk})") + lines.append("") + lines.append(" PTR ok forward-confirmed: PTR ends in a vendor domain AND") + lines.append(" that hostname resolves back to the same address.") + lines.append(" ASN ok no PTR, but the address sits in an ASN belonging") + lines.append(" solely to the vendor (Meta's AS32934). Only used") + lines.append(" where the ASN sells no compute - never for clouds.") + lines.append(" FORGED a PTR exists and points somewhere else. This is") + lines.append(" the only column that is evidence of forgery.") + lines.append(" no-PTR no PTR record and no usable ASN: unknown, not bad.") + if sum(rdns_private.values()): + lines.append(" priv logged source was RFC1918: a proxy or Docker") + lines.append(" bridge masked the client. Unjudgeable. See README.") + + # --- what the verified crawlers actually fetched --------------------- + # Last, because it spans both tables above: an address only counts here + # once it has been confirmed, whether by IP range, PTR or ASN. + if ver_paths: + lines.append("") + lines.append("=" * 62) + lines.append("WHAT THE VERIFIED CRAWLERS DID (all methods)") + lines.append("=" * 62) + tot_v = sum(ver_paths.values()) + mb = ver_bytes[0] / 1048576 + lines.append(f" {tot_v} requests, {mb:.1f} MB served.") + if ver_status: + mix = " ".join(f"{s}:{c}" for s, c in ver_status.most_common(6)) + lines.append(f" Status: {mix}") + if ver_bot_bytes: + top = " ".join(f"{b} {v/1048576:.1f}MB" + for b, v in ver_bot_bytes.most_common(4)) + lines.append(f" Bandwidth: {top}") + lines.append("") + if not ROBOTS_RULES: + lines.append(" robots.txt compliance: not checked " + "(set CRAWLER_ROBOTS to enable)") + elif ver_violations: + n = sum(ver_violations.values()) + lines.append(f" ROBOTS.TXT VIOLATIONS: {n} requests to disallowed paths") + for p, c in ver_violations.most_common(6): + lines.append(f" {c:>6} {p}") + lines.append(" A verified crawler ignoring robots.txt is a policy") + lines.append(" problem, not a security one - the identity is genuine.") + else: + lines.append(f" robots.txt compliance: clean, 0 violations in " + f"{tot_v} requests") + lines.append("") + # Repository content vs everything else. If robots.txt permits + # /src/ and /raw/, this is what says whether anyone took it up. + nc = sum(ver_content.values()) + pct_c = nc / tot_v * 100 if tot_v else 0 + lines.append(f" Repository content fetched: {nc} of {tot_v} requests " + f"({pct_c:.1f}%)") + if nc: + by_bot = " ".join(f"{b} {c}" for b, c in ver_content.most_common(5)) + lines.append(f" by: {by_bot}") + for p, c in ver_content_paths.most_common(8): + lines.append(f" {c:>6} {p}") + else: + lines.append(" none - crawlers indexed the site but read no code") + # Which repositories each vendor cares about. Grouped by vendor + # rather than by UA token: OpenAI alone sends three tokens, and + # "what is OpenAI reading" is the useful question. + if ver_repos: + groups = {} + for b, c in ver_repos.items(): + if b in BOT_VENDOR: + v = BOT_VENDOR[b] # openai, anthropic, ... + elif b in BOT_ASN: + v = "meta" + elif b in BOT_RDNS: + v = b.lower() # amazonbot, youbot, ... + else: + v = b.lower() + groups.setdefault(v, Counter()).update(c) + lines.append("") + lines.append(" Repositories by vendor:") + for v in sorted(groups, key=lambda x: -sum(groups[x].values())): + tot = sum(groups[v].values()) + top = ", ".join(f"{r} ({n})" for r, n in groups[v].most_common(3)) + lines.append(f" {v:<12} {tot:>5} hits {top}") + if ver_pkgs or ver_pkg_index: + np = sum(ver_pkgs.values()) + sum(ver_pkg_index.values()) + lines.append("") + lines.append(f" Package registry: {np} requests") + for p, c in ver_pkgs.most_common(6): + lines.append(f" {c:>6} {p}") + if ver_pkg_index: + idx = ", ".join(f"{o} ({n})" for o, n in ver_pkg_index.most_common(4)) + lines.append(f" index pages: {idx}") + lines.append("") + lines.append(" Top paths overall:") + for p, c in ver_paths.most_common(12): + lines.append(f" {c:>6} {p}") + body = "\n".join(lines) + "\n" + subject = f"{SITE} weekly crawler report: {total} requests, {len(all_ips)} IPs" + if DRY: + print(subject); print(); print(body) + sys.exit(0) + +# SMTP settings: use CRAWLER_SMTP_* if given, otherwise borrow them from +# a Forgejo/Gitea app.ini [mailer] section so there is no second copy of +# the credentials to keep in sync. +if os.environ.get("CRAWLER_SMTP_ADDR"): + addr = os.environ["CRAWLER_SMTP_ADDR"] + port = int(os.environ.get("CRAWLER_SMTP_PORT", "587")) + user = os.environ.get("CRAWLER_SMTP_USER", "") + passwd = os.environ.get("CRAWLER_SMTP_PASSWORD", "") + sender = os.environ.get("CRAWLER_SMTP_FROM", user or RECIPIENT) + proto = os.environ.get("CRAWLER_SMTP_PROTOCOL", "smtp+starttls") +else: + ini = subprocess.run(["docker", "exec", MAIL_CONTAINER, "cat", MAIL_INI], + capture_output=True, text=True).stdout + if not ini.strip(): + sys.exit(f"could not read {MAIL_INI} from container " + f"'{MAIL_CONTAINER}' and CRAWLER_SMTP_ADDR is unset") + cp = configparser.ConfigParser(interpolation=None, strict=False) + cp.read_string("[DEFAULT]\n" + ini) + m = cp["mailer"] + proto = m.get("PROTOCOL", "smtps").strip() + addr, port = m.get("SMTP_ADDR").strip(), int(m.get("SMTP_PORT", "465").strip()) + user, passwd = m.get("USER", "").strip(), m.get("PASSWD", "").strip().strip("`\"") + sender = m.get("FROM", user).strip() + +msg = MIMEText(body) +msg["Subject"], msg["From"], msg["To"] = subject, sender, RECIPIENT +if proto == "smtps": + s = smtplib.SMTP_SSL(addr, port, timeout=30) +else: + s = smtplib.SMTP(addr, port, timeout=30) + if proto != "smtp": # plain smtp = no TLS (local relay) + s.starttls() +if user: # unauthenticated local relays exist + s.login(user, passwd) +s.sendmail(sender, [RECIPIENT], msg.as_string()) +s.quit() +print(f"mailed to {RECIPIENT}") diff --git a/ansible/roles/crawler_alert/files/crawler-alert.service b/ansible/roles/crawler_alert/files/crawler-alert.service new file mode 100644 index 0000000..6df88b7 --- /dev/null +++ b/ansible/roles/crawler_alert/files/crawler-alert.service @@ -0,0 +1,8 @@ +[Unit] +Description=Daily crawler threshold alert +After=docker.service + +[Service] +Type=oneshot +EnvironmentFile=/etc/crawler-alert.env +ExecStart=/usr/local/bin/crawler-alert.py diff --git a/ansible/roles/crawler_alert/files/crawler-alert.timer b/ansible/roles/crawler_alert/files/crawler-alert.timer new file mode 100644 index 0000000..0b8b4f5 --- /dev/null +++ b/ansible/roles/crawler_alert/files/crawler-alert.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Daily Forgejo crawler volume check + +[Timer] +OnCalendar=*-*-* 00:15:00 +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/crawler_alert/files/crawler-weekly.service b/ansible/roles/crawler_alert/files/crawler-weekly.service new file mode 100644 index 0000000..ed9916f --- /dev/null +++ b/ansible/roles/crawler_alert/files/crawler-weekly.service @@ -0,0 +1,8 @@ +[Unit] +Description=Weekly crawler report mail +After=docker.service + +[Service] +Type=oneshot +EnvironmentFile=/etc/crawler-alert.env +ExecStart=/usr/local/bin/crawler-alert.py --weekly diff --git a/ansible/roles/crawler_alert/files/crawler-weekly.timer b/ansible/roles/crawler_alert/files/crawler-weekly.timer new file mode 100644 index 0000000..2982024 --- /dev/null +++ b/ansible/roles/crawler_alert/files/crawler-weekly.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Weekly Forgejo crawler report, Mondays 12:00 UTC + +[Timer] +OnCalendar=Mon *-*-* 12:00:00 +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/crawler_alert/handlers/main.yaml b/ansible/roles/crawler_alert/handlers/main.yaml new file mode 100644 index 0000000..c28484f --- /dev/null +++ b/ansible/roles/crawler_alert/handlers/main.yaml @@ -0,0 +1,4 @@ +--- +- name: Reload systemd + ansible.builtin.systemd: + daemon_reload: true diff --git a/ansible/roles/crawler_alert/tasks/main.yaml b/ansible/roles/crawler_alert/tasks/main.yaml new file mode 100644 index 0000000..85cd559 --- /dev/null +++ b/ansible/roles/crawler_alert/tasks/main.yaml @@ -0,0 +1,37 @@ +--- +- name: Install crawler-alert script + ansible.builtin.copy: + src: crawler-alert.py + dest: /usr/local/bin/crawler-alert.py + mode: "0755" + +- name: Configure crawler-alert + ansible.builtin.template: + src: crawler-alert.env.j2 + dest: /etc/crawler-alert.env + mode: "0600" + diff: false + +- name: Install crawler-alert units + ansible.builtin.copy: + src: "{{ item }}" + dest: "/etc/systemd/system/{{ item }}" + mode: "0644" + loop: + - crawler-alert.service + - crawler-alert.timer + - crawler-weekly.service + - crawler-weekly.timer + notify: Reload systemd + +- name: Flush handlers + ansible.builtin.meta: flush_handlers + +- name: Enable crawler-alert timers + ansible.builtin.systemd: + name: "{{ item }}" + enabled: true + state: started + loop: + - crawler-alert.timer + - crawler-weekly.timer diff --git a/ansible/roles/crawler_alert/templates/crawler-alert.env.j2 b/ansible/roles/crawler_alert/templates/crawler-alert.env.j2 new file mode 100644 index 0000000..7677840 --- /dev/null +++ b/ansible/roles/crawler_alert/templates/crawler-alert.env.j2 @@ -0,0 +1,8 @@ +CRAWLER_RECIPIENT={{ crawler_recipient }} +CRAWLER_SITE={{ crawler_site }} +CRAWLER_LOG=/opt/traefik/logs/access.log* +CRAWLER_ROUTER=forgejo@docker +CRAWLER_THRESHOLD={{ crawler_threshold }} +CRAWLER_MAIL_CONTAINER=forgejo +CRAWLER_MAIL_INI=/data/gitea/conf/app.ini +CRAWLER_ROBOTS={{ crawler_robots_url }} diff --git a/ansible/roles/docker_service/defaults/main.yaml b/ansible/roles/docker_service/defaults/main.yaml index 3f9d50b..b3baa37 100644 --- a/ansible/roles/docker_service/defaults/main.yaml +++ b/ansible/roles/docker_service/defaults/main.yaml @@ -1,34 +1,28 @@ --- -# Default variables for docker_service role -# These can be overridden when including this role +service_name: "" # required, also the compose project name +service_base_path: "" # required, e.g. /opt/traefik -# Service configuration -service_name: "" # REQUIRED: Name of the service (e.g., "traefik", "forgejo") -service_base_path: "" # REQUIRED: Base installation path (e.g., "/opt/traefik") - -# Directories to create under service_base_path -# Format: list of dicts with 'path' and 'mode' keys +# Each list item: path/dest plus optional owner, group, mode. Ownership and +# mode are only enforced when given, so existing data directories keep theirs. service_directories: [] -# Example: -# - { path: "data", mode: "0755" } -# - { path: "config", mode: "0644" } +service_templates: [] # src (template), dest +service_files: [] # src (plain file, copied verbatim), dest +service_absent: [] # paths under the base path that must not exist -# Template files to deploy -# Format: list of dicts with 'src', 'dest', and optional 'mode' keys -service_templates: [] -# Example: -# - { src: "docker-compose.yml.j2", dest: "docker-compose.yml", mode: "0644" } -# - { src: "config.yaml.j2", dest: "config/config.yaml", mode: "0600" } +# Containers to save and stop properly before compose recreates them: +# - { container: name, timeout: 120, pre_stop: "docker exec ..." } +service_graceful_stop: [] -# Docker compose configuration -docker_compose_file: "docker-compose.yml" # Name of the compose file -docker_compose_project_name: "{{ service_name }}" # Docker compose project name +docker_compose_file: "docker-compose.yml" +docker_compose_project_name: "{{ service_name }}" -# Health check configuration -health_check_enabled: true # Whether to perform health check after deployment -health_check_url: "" # URL to check (empty = skip HTTP check) -health_check_host: "localhost" # Host to check port on (default: localhost) -health_check_port: "" # Port to check (empty = skip port check) -health_check_delay: 10 # Seconds to wait before health check -health_check_retries: 30 # Number of retries for health check -health_check_retry_delay: 2 # Seconds between retries +# A changed config file restarts the stack unless compose already recreated it +service_restart_on_change: true + +health_check_enabled: true +health_check_url: "" +health_check_host: "localhost" +health_check_port: "" +health_check_delay: 10 +health_check_retries: 30 +health_check_retry_delay: 2 diff --git a/ansible/roles/docker_service/tasks/main.yaml b/ansible/roles/docker_service/tasks/main.yaml index 5e3e7d0..78eed78 100644 --- a/ansible/roles/docker_service/tasks/main.yaml +++ b/ansible/roles/docker_service/tasks/main.yaml @@ -1,46 +1,94 @@ --- -# Shared tasks for deploying Docker-based services -# This role provides a standardized pattern for service deployment - - name: Validate required variables ansible.builtin.assert: that: - - service_name is defined - service_name | length > 0 - - service_base_path is defined - service_base_path | length > 0 - fail_msg: "service_name and service_base_path are required variables" - success_msg: "Required variables validated for {{ service_name }}" + fail_msg: "service_name and service_base_path are required" + quiet: true - name: Create base service directory ansible.builtin.file: path: "{{ service_base_path }}" state: directory - mode: '0755' - owner: root - group: root - name: Create service subdirectories ansible.builtin.file: path: "{{ service_base_path }}/{{ item.path }}" state: directory - mode: "{{ item.mode | default('0755') }}" - owner: root - group: root + owner: "{{ item.owner | default(omit) }}" + group: "{{ item.group | default(omit) }}" + mode: "{{ item.mode | default(omit) }}" loop: "{{ service_directories }}" - when: service_directories | length > 0 + loop_control: + label: "{{ item.path }}" -- name: Deploy service template files +- name: Deploy service templates ansible.builtin.template: src: "{{ item.src }}" dest: "{{ service_base_path }}/{{ item.dest }}" + owner: "{{ item.owner | default('root') }}" + group: "{{ item.group | default('root') }}" mode: "{{ item.mode | default('0644') }}" - owner: root - group: root loop: "{{ service_templates }}" - when: service_templates | length > 0 + loop_control: + label: "{{ item.dest }}" + # Templates may carry secrets; never print their contents + diff: false register: template_result +- name: Deploy service files + ansible.builtin.copy: + src: "{{ item.src }}" + dest: "{{ service_base_path }}/{{ item.dest }}" + owner: "{{ item.owner | default('root') }}" + group: "{{ item.group | default('root') }}" + mode: "{{ item.mode | default('0644') }}" + loop: "{{ service_files }}" + loop_control: + label: "{{ item.dest }}" + register: file_result + +- name: Remove files that are no longer part of the service + ansible.builtin.file: + path: "{{ service_base_path }}/{{ item }}" + state: absent + loop: "{{ service_absent }}" + +# Game servers need time to save: Docker's default stop timeout is 10 s. +# When compose is about to recreate one of the listed containers, save +# and stop it properly first; compose then replaces the stopped container. +- name: Ask compose what it would do + ansible.builtin.command: + cmd: docker compose -p {{ docker_compose_project_name }} -f {{ docker_compose_file }} up -d --dry-run + chdir: "{{ service_base_path }}" + register: compose_plan + changed_when: false + # The dry run exits non-zero when an image still has to be pulled; the + # plan it prints up to that point is all that is needed here + failed_when: false + check_mode: false + when: service_graceful_stop | length > 0 + +- name: Run pre-stop command before recreation + ansible.builtin.command: + cmd: "{{ item.pre_stop }}" + loop: "{{ service_graceful_stop }}" + loop_control: + label: "{{ item.container }}" + when: + - item.pre_stop is defined + - (compose_plan.stdout ~ compose_plan.stderr) is search('Container ' ~ item.container ~ ' Recreate') + failed_when: false + +- name: Stop gracefully before recreation + ansible.builtin.command: + cmd: "docker stop -t {{ item.timeout | default(120) }} {{ item.container }}" + loop: "{{ service_graceful_stop }}" + loop_control: + label: "{{ item.container }}" + when: (compose_plan.stdout ~ compose_plan.stderr) is search('Container ' ~ item.container ~ ' Recreate') + - name: Start service with Docker Compose community.docker.docker_compose_v2: project_src: "{{ service_base_path }}" @@ -49,33 +97,45 @@ - "{{ docker_compose_file }}" state: present register: compose_result + # In check mode compose cannot pull, so a missing image looks like an error + ignore_errors: "{{ ansible_check_mode }}" + +- name: Restart service after a config change + community.docker.docker_compose_v2: + project_src: "{{ service_base_path }}" + project_name: "{{ docker_compose_project_name }}" + files: + - "{{ docker_compose_file }}" + state: restarted + when: + - service_restart_on_change + - template_result is changed or file_result is changed + - compose_result is not changed - name: Wait before health check ansible.builtin.pause: seconds: "{{ health_check_delay }}" - when: health_check_enabled and (health_check_url != "" or health_check_port != "") + when: + - health_check_enabled + - health_check_url != "" or health_check_port != "" + - compose_result is changed or template_result is changed or file_result is changed - name: Wait for service HTTP endpoint ansible.builtin.uri: url: "{{ health_check_url }}" - status_code: [200, 301, 302, 401, 404] # Accept various codes (service-dependent) + status_code: [200, 301, 302, 303, 401, 404] + follow_redirects: none timeout: 5 register: health_result until: health_result is not failed retries: "{{ health_check_retries }}" delay: "{{ health_check_retry_delay }}" when: health_check_enabled and health_check_url != "" - failed_when: false + check_mode: false - name: Wait for service port ansible.builtin.wait_for: host: "{{ health_check_host }}" port: "{{ health_check_port }}" - delay: 0 timeout: "{{ health_check_retries * health_check_retry_delay }}" - state: started when: health_check_enabled and health_check_port != "" - -- name: Display deployment status - ansible.builtin.debug: - msg: "{{ service_name }} deployment completed successfully" diff --git a/ansible/roles/forgejo/tasks/main.yaml b/ansible/roles/forgejo/tasks/main.yaml index aac34c8..541a5af 100644 --- a/ansible/roles/forgejo/tasks/main.yaml +++ b/ansible/roles/forgejo/tasks/main.yaml @@ -5,19 +5,22 @@ - name: Configure SSH passthrough ansible.builtin.import_tasks: ssh-passthrough.yaml -- name: Deploy Forgejo using shared docker_service role +- name: Deploy Forgejo ansible.builtin.include_role: name: docker_service vars: service_name: "forgejo" service_base_path: "{{ forgejo_base_path }}" service_directories: - - { path: "data", mode: "0755" } - - { path: "config", mode: "0755" } + - { path: "data" } + - { path: "config" } service_templates: - - { src: "app.ini.j2", dest: "config/app.ini", mode: "0644" } - - { src: "docker-compose.yml.j2", dest: "docker-compose.yml", mode: "0644" } - health_check_enabled: true - health_check_url: "http://localhost:{{ forgejo_http_port }}/" + - src: "app.ini.j2" + dest: "config/app.ini" + owner: "{{ forgejo_uid }}" + group: "{{ forgejo_gid }}" + mode: "0600" + - { src: "docker-compose.yml.j2", dest: "docker-compose.yml" } + health_check_url: "http://localhost:{{ forgejo_http_port }}/api/v1/version" health_check_delay: 5 - health_check_retries: 15 + health_check_retries: 30 diff --git a/ansible/roles/forgejo/tasks/ssh-passthrough.yaml b/ansible/roles/forgejo/tasks/ssh-passthrough.yaml index b95b3eb..ffcb436 100644 --- a/ansible/roles/forgejo/tasks/ssh-passthrough.yaml +++ b/ansible/roles/forgejo/tasks/ssh-passthrough.yaml @@ -1,45 +1,32 @@ --- +# git@ lands on the host's sshd, which hands the session to the +# Forgejo container. Keys are looked up through the container too +# (AuthorizedKeysCommand), so the host keeps no authorized_keys for git. + - name: Create git user on host - user: + ansible.builtin.user: name: git comment: "Forgejo Git User" - home: /var/lib/git + home: "{{ forgejo_base_path }}/data/git" + create_home: false shell: /usr/local/bin/forgejo-shell - system: yes - create_home: yes + system: true + groups: docker + append: true - name: Deploy forgejo-shell wrapper script - template: + ansible.builtin.template: src: forgejo-shell.j2 dest: /usr/local/bin/forgejo-shell owner: root group: root - mode: '0755' - -- name: Create .ssh directory for git user - file: - path: /var/lib/git/.ssh - state: directory - owner: git - group: git - mode: '0700' + mode: "0755" - name: Deploy SSH daemon configuration for Forgejo passthrough - template: + ansible.builtin.template: src: sshd-forgejo.conf.j2 dest: /etc/ssh/sshd_config.d/forgejo.conf owner: root group: root - mode: '0644' + mode: "0644" notify: Reload SSH daemon - -- name: Ensure docker group exists - group: - name: docker - state: present - -- name: Add git user to docker group - user: - name: git - groups: docker - append: yes diff --git a/ansible/roles/forgejo/templates/app.ini.j2 b/ansible/roles/forgejo/templates/app.ini.j2 index 42b363f..0220189 100644 --- a/ansible/roles/forgejo/templates/app.ini.j2 +++ b/ansible/roles/forgejo/templates/app.ini.j2 @@ -1,5 +1,6 @@ -APP_NAME = Gitea: Git with a cup of tea together with asxpi +APP_NAME = Git with a cup of tea RUN_MODE = prod +WORK_PATH = /data/gitea [server] PROTOCOL = http @@ -13,10 +14,12 @@ SSH_LISTEN_PORT = 22 START_SSH_SERVER = false LFS_START_SERVER = true OFFLINE_MODE = false +LFS_JWT_SECRET = {{ forgejo_server_lfs_jwt_secret }} +LANDING_PAGE = explore [database] DB_TYPE = sqlite3 -PATH = /data/gitea.db +PATH = /data/gitea/gitea.db LOG_SQL = false [repository] @@ -26,7 +29,7 @@ DEFAULT_BRANCH = main DEFAULT_PRIVATE = private ENABLE_PUSH_CREATE_USER = true ENABLE_PUSH_CREATE_ORG = false -DISABLED_REPO_UNITS = +DISABLED_REPO_UNITS = DEFAULT_REPO_UNITS = repo.code,repo.releases,repo.issues,repo.pulls,repo.wiki,repo.projects,repo.packages,repo.actions [ui] @@ -35,15 +38,15 @@ DEFAULT_THEME = forgejo-auto [security] INSTALL_LOCK = true -SECRET_KEY = {{ forgejo_secret_key | default('CHANGE-ME-PLEASE-GENERATE-NEW-SECRET') }} -INTERNAL_TOKEN = {{ forgejo_internal_token | default('CHANGE-ME-PLEASE-GENERATE-NEW-TOKEN') }} +SECRET_KEY = {{ forgejo_secret_key }} +INTERNAL_TOKEN = {{ forgejo_internal_token }} PASSWORD_HASH_ALGO = pbkdf2 [service] DISABLE_REGISTRATION = true REQUIRE_SIGNIN_VIEW = false REGISTER_EMAIL_CONFIRM = false -ENABLE_NOTIFY_MAIL = false +ENABLE_NOTIFY_MAIL = true DEFAULT_KEEP_EMAIL_PRIVATE = true DEFAULT_ALLOW_CREATE_ORGANIZATION = true DEFAULT_ENABLE_TIMETRACKING = true @@ -51,10 +54,16 @@ NO_REPLY_ADDRESS = noreply.{{ forgejo_domain }} [oauth2] ENABLED = true -JWT_SECRET = {{ forgejo_jwt_secret | default('CHANGE-ME-PLEASE-GENERATE-NEW-JWT-SECRET') }} +JWT_SECRET = {{ forgejo_jwt_secret }} [mailer] -ENABLED = false +ENABLED = true +FROM = Czechoslovakia Git +PROTOCOL = smtp+starttls +SMTP_ADDR = smtp.fastmail.com +SMTP_PORT = 587 +USER = {{ forgejo_smtp_user }} +PASSWD = {{ forgejo_smtp_password }} [session] PROVIDER = file @@ -80,4 +89,13 @@ DEFAULT_TRUST_MODEL = collaborator [lfs] PATH = /data/git/lfs -LFS_JWT_SECRET = {{ forgejo_lfs_jwt_secret | default('CHANGE-ME-PLEASE-GENERATE-NEW-LFS-JWT-SECRET') }} +LFS_JWT_SECRET = {{ forgejo_lfs_jwt_secret }} + +[packages] +ENABLED = true + +[other] +SHOW_FOOTER_VERSION = false +SHOW_FOOTER_TEMPLATE_LOAD_TIME = false +SHOW_FOOTER_POWERED_BY = false + diff --git a/ansible/roles/forgejo/templates/docker-compose.yml.j2 b/ansible/roles/forgejo/templates/docker-compose.yml.j2 index 0bcc023..e4e114c 100644 --- a/ansible/roles/forgejo/templates/docker-compose.yml.j2 +++ b/ansible/roles/forgejo/templates/docker-compose.yml.j2 @@ -19,7 +19,7 @@ services: - {{ traefik_network }} labels: - "traefik.enable=true" - - "traefik.http.routers.forgejo.rule=Host(`{{ forgejo_domain }}`)" + - "traefik.http.routers.forgejo.rule={{ ([forgejo_domain] + forgejo_alias_domains) | map('regex_replace', '^(.*)$', 'Host(`\\1`)') | join(' || ') }}" - "traefik.http.routers.forgejo.entrypoints=websecure" - "traefik.http.routers.forgejo.tls=true" - "traefik.http.routers.forgejo.tls.certresolver=letsencrypt" diff --git a/ansible/roles/forgejo_backup/tasks/main.yaml b/ansible/roles/forgejo_backup/tasks/main.yaml deleted file mode 100644 index 9b5d8ff..0000000 --- a/ansible/roles/forgejo_backup/tasks/main.yaml +++ /dev/null @@ -1,102 +0,0 @@ ---- -- name: Create backup directory on ops server - file: - path: "{{ backup_dest_dir }}" - state: directory - mode: '0755' - owner: asxpi - group: asxpi - delegate_to: asxpio-ops - become: true - -- name: Find existing Forgejo dump files on old server - find: - paths: /srv/gitea - patterns: "*.zip" - file_type: file - register: existing_dumps - -- name: Create temporary staging directory for backup - file: - path: /tmp/forgejo-backup-staging - state: directory - mode: '0755' - become: true - -- name: Copy database to staging - copy: - src: /srv/gitea/gitea.db - dest: /tmp/forgejo-backup-staging/gitea.db - remote_src: true - become: true - -- name: Create data directory in staging - file: - path: /tmp/forgejo-backup-staging/data - state: directory - mode: '0755' - become: true - -- name: Copy data directories to staging - copy: - src: "/srv/gitea/data/{{ item }}/" - dest: "/tmp/forgejo-backup-staging/data/{{ item }}/" - remote_src: true - loop: - - avatars - - repo-avatars - - repo-archive - - attachments - - gitea-repositories - become: true - ignore_errors: true - -- name: Create backup archive from staging - archive: - path: /tmp/forgejo-backup-staging - dest: /tmp/forgejo-full-backup-{{ ansible_date_time.date }}.tar.gz - format: gz - become: true - -- name: Clean up staging directory - file: - path: /tmp/forgejo-backup-staging - state: absent - become: true - -- name: Display found backup files - debug: - msg: | - Found {{ existing_dumps.files | length }} dump files - Created full backup archive: /tmp/forgejo-full-backup-{{ ansible_date_time.date }}.tar.gz - -- name: Fetch dump file to ops server (if exists) - synchronize: - mode: pull - src: "{{ (existing_dumps.files | sort(attribute='mtime', reverse=true) | first).path }}" - dest: /opt/forgejo-backup/ - delegate_to: asxpio-ops - when: existing_dumps.files | length > 0 - ignore_errors: true - -- name: Fetch full backup archive to ops server - synchronize: - mode: pull - src: /tmp/forgejo-full-backup-{{ ansible_date_time.date }}.tar.gz - dest: /opt/forgejo-backup/ - delegate_to: asxpio-ops - become: true - -- name: Display backup location - debug: - msg: | - Backup files transferred to asxpio-ops:/opt/forgejo-backup/ - - Files: - - forgejo-full-backup-{{ ansible_date_time.date }}.tar.gz (complete backup) - {% if existing_dumps.files | length > 0 %} - - {{ (existing_dumps.files | sort(attribute='mtime', reverse=true) | first).path | basename }} (dump file) - {% endif %} - - Next step: Run restore playbook - ansible-playbook playbooks/restore-forgejo.yaml diff --git a/ansible/roles/forgejo_restore/tasks/main.yaml b/ansible/roles/forgejo_restore/tasks/main.yaml deleted file mode 100644 index f3faceb..0000000 --- a/ansible/roles/forgejo_restore/tasks/main.yaml +++ /dev/null @@ -1,142 +0,0 @@ ---- -- name: Find backup files on server - find: - paths: "{{ backup_dest_dir }}" - patterns: "forgejo-full-backup-*.tar.gz" - register: server_backup_files - -- name: Fail if no backup files found - fail: - msg: "No backup files found in {{ backup_dest_dir }}. Please run backup-forgejo.yaml first." - when: server_backup_files.files | length == 0 - -- name: Select most recent backup file - set_fact: - backup_file: "{{ (server_backup_files.files | sort(attribute='mtime', reverse=true) | first).path }}" - -- name: Display backup file to be restored - debug: - msg: "Restoring from: {{ backup_file }}" - -- name: Stop Forgejo container - community.docker.docker_compose_v2: - project_src: /opt/forgejo - state: stopped - ignore_errors: true - -- name: Backup current Forgejo data (if exists) - archive: - path: /opt/forgejo/data - dest: "/opt/forgejo/data-backup-{{ ansible_date_time.epoch }}.tar.gz" - format: gz - ignore_errors: true - -- name: Remove old data directory - file: - path: /opt/forgejo/data - state: absent - -- name: Create fresh data directory structure - file: - path: "{{ item.path }}" - state: directory - owner: 1000 - group: 1000 - mode: "{{ item.mode | default('0755') }}" - loop: - - { path: '/opt/forgejo/data' } - - { path: '/opt/forgejo/data/gitea' } - - { path: '/opt/forgejo/data/gitea/data' } - - { path: '/opt/forgejo/data/git' } - - { path: '/opt/forgejo/data/git/repositories' } - - { path: '/opt/forgejo/data/git/.ssh', mode: '0700' } - -- name: Create temporary restore directory - file: - path: /tmp/forgejo-restore - state: directory - mode: '0755' - -- name: Extract backup to temporary location - unarchive: - src: "{{ backup_file }}" - dest: /tmp/forgejo-restore/ - remote_src: true - -- name: Set backup source path - set_fact: - backup_source: "/tmp/forgejo-restore/forgejo-backup-staging" - -- name: Copy database from backup - copy: - src: "{{ backup_source }}/gitea.db" - dest: /opt/forgejo/data/gitea/gitea.db - remote_src: true - owner: 1000 - group: 1000 - mode: '0644' - -- name: Copy data directories from backup - ansible.builtin.copy: - src: "{{ backup_source }}/data/{{ item.src }}/" - dest: "{{ forgejo_base_path | default('/opt/forgejo') }}/data/{{ item.dest }}/" - remote_src: true - owner: 1000 - group: 1000 - mode: preserve - loop: - - { src: "avatars", dest: "gitea/data/avatars" } - - { src: "repo-avatars", dest: "gitea/data/repo-avatars" } - - { src: "repo-archive", dest: "gitea/data/repo-archive" } - - { src: "attachments", dest: "gitea/data/attachments" } - - { src: "gitea-repositories", dest: "git/repositories" } - ignore_errors: "{{ item.src != 'gitea-repositories' }}" # Only fail on repositories copy failure - -- name: Set correct ownership for all Forgejo data - ansible.builtin.file: - path: "{{ forgejo_base_path | default('/opt/forgejo') }}/data" - owner: 1000 - group: 1000 - recurse: true - -- name: Start Forgejo container - community.docker.docker_compose_v2: - project_src: "{{ forgejo_base_path | default('/opt/forgejo') }}" - state: present - -- name: Wait for Forgejo to be healthy - wait_for: - host: localhost - port: "{{ forgejo_http_port }}" - timeout: 60 - delay: 5 - -- name: Clean up temporary files - file: - path: /tmp/forgejo-restore - state: absent - -- name: Verify users in database - ansible.builtin.command: docker exec -u git forgejo forgejo admin user list - register: user_list - changed_when: false - ignore_errors: true - -- name: Display restore information - debug: - msg: | - Forgejo restore completed successfully! - - Users: - {{ user_list.stdout_lines | join('\n ') }} - - Forgejo is now running at: - - Internal: http://{{ asxpio_ops_wireguard_ip }}:{{ forgejo_http_port }} - - External: https://{{ forgejo_domain }} - - Next steps: - 1. Test web interface: https://{{ forgejo_domain }} - 2. Verify avatars and images are visible - 3. Test login with your credentials - 4. Test git clone: git clone https://{{ forgejo_domain }}/user/repo.git - 5. Check logs: docker logs forgejo diff --git a/ansible/roles/forgejo_runner/tasks/main.yaml b/ansible/roles/forgejo_runner/tasks/main.yaml index f1e868b..0269082 100644 --- a/ansible/roles/forgejo_runner/tasks/main.yaml +++ b/ansible/roles/forgejo_runner/tasks/main.yaml @@ -9,12 +9,12 @@ service_name: "forgejo-runner" service_base_path: "{{ forgejo_runner_base_path }}" service_directories: - - { path: "data", mode: "0755" } - - { path: "config", mode: "0755" } + - { path: "data" } + - { path: "config" } service_templates: - - { src: "config.yaml.j2", dest: "config/config.yaml", mode: "0644" } - - { src: "docker-compose.yml.j2", dest: "docker-compose.yml", mode: "0644" } - health_check_enabled: false # Runner doesn't expose HTTP port + - { src: "config.yaml.j2", dest: "config/config.yaml" } + - { src: "docker-compose.yml.j2", dest: "docker-compose.yml" } + health_check_enabled: false # the runner exposes no port - name: Check if runner is already registered ansible.builtin.stat: diff --git a/ansible/roles/minecraft/files/docker-compose.yml b/ansible/roles/minecraft/files/docker-compose.yml new file mode 100644 index 0000000..5251913 --- /dev/null +++ b/ansible/roles/minecraft/files/docker-compose.yml @@ -0,0 +1,15 @@ +services: + minecraft: + image: eclipse-temurin:21-jre + container_name: minecraft + working_dir: /data + command: java -Xmx4G -Xms2G -jar paper-1.21.11-54.jar nogui + stdin_open: true + tty: true + restart: unless-stopped + stop_grace_period: 2m + ports: + - "25565:25565" + - "19132:19132/udp" + volumes: + - /opt/minecraft:/data diff --git a/ansible/roles/minecraft/tasks/main.yaml b/ansible/roles/minecraft/tasks/main.yaml new file mode 100644 index 0000000..6656822 --- /dev/null +++ b/ansible/roles/minecraft/tasks/main.yaml @@ -0,0 +1,15 @@ +--- +# Paper server; the world and the server jar live next to the compose +# file in /opt/minecraft and are not managed here. A compose change +# recreates the container, so it is stopped with time to save first. +- name: Deploy Minecraft + ansible.builtin.include_role: + name: docker_service + vars: + service_name: "minecraft" + service_base_path: "{{ services_base_path }}/minecraft" + service_files: + - { src: "docker-compose.yml", dest: "docker-compose.yml" } + service_graceful_stop: + - { container: minecraft, timeout: 120 } + health_check_enabled: false diff --git a/ansible/roles/minecraft_fossils/files/docker-compose.yml b/ansible/roles/minecraft_fossils/files/docker-compose.yml new file mode 100644 index 0000000..c937a27 --- /dev/null +++ b/ansible/roles/minecraft_fossils/files/docker-compose.yml @@ -0,0 +1,87 @@ +services: + minecraft-fossils: + image: itzg/minecraft-server:java21 + container_name: minecraft-fossils + restart: unless-stopped + stop_grace_period: 2m + stdin_open: true + tty: true + ports: + - "25566:25565" + env_file: + - secrets.env + environment: + EULA: "TRUE" + TYPE: FABRIC + VERSION: "1.20.1" + FABRIC_LOADER_VERSION: "0.18.4" + MEMORY: "8G" + ONLINE_MODE: "FALSE" + ENFORCE_WHITELIST: "TRUE" + ENFORCE_SECURE_PROFILE: "FALSE" + WHITELIST_FILE: /data/whitelist.json + OPS_FILE: /data/ops.json + ENABLE_RCON: "TRUE" + MOTD: "[mc.czsk.dev] Fossils Archaeology Revival: Genesis" + USE_AIKAR_FLAGS: "TRUE" + volumes: + - /opt/minecraft-fossils/data:/data + networks: + - fossils + + whitelist-sync: + image: alpine:3 + container_name: minecraft-fossils-wlsync + restart: unless-stopped + depends_on: + - minecraft-fossils + volumes: + - /opt/minecraft:/src:ro + - /opt/minecraft-fossils/data:/dst + env_file: + - secrets.env + environment: + RCON_HOST: minecraft-fossils + INTERVAL: "15" + command: + - sh + - -c + - | + apk add --no-cache python3 >/dev/null + cat > /rcon.py <<'PY' + import socket, struct, sys, os + host, port, pw, cmd = os.environ["RCON_HOST"], 25575, os.environ["RCON_PASSWORD"], sys.argv[1] + def pkt(i, t, b): p = struct.pack("/dev/null; then + cp /src/$$f /dst/$$f + echo "[wlsync] copied $$f" + case $$f in + whitelist.json) CMD="whitelist reload" ;; + ops.json) CMD="reload" ;; + banned-players.json|banned-ips.json) CMD="kick @a Ban list updated" ;; + esac + python3 /rcon.py "$$CMD" 2>&1 | sed 's/^/[wlsync] rcon: /' || echo "[wlsync] rcon failed" + fi + done + sleep $$INTERVAL + done + networks: + - fossils + +networks: + fossils: + driver: bridge diff --git a/ansible/roles/minecraft_fossils/files/minecraft-fossils-restart.service b/ansible/roles/minecraft_fossils/files/minecraft-fossils-restart.service new file mode 100644 index 0000000..7798cd6 --- /dev/null +++ b/ansible/roles/minecraft_fossils/files/minecraft-fossils-restart.service @@ -0,0 +1,8 @@ +[Unit] +Description=Daily restart of minecraft-fossils container +Requires=docker.service +After=docker.service + +[Service] +Type=oneshot +ExecStart=/usr/bin/docker restart minecraft-fossils diff --git a/ansible/roles/minecraft_fossils/files/minecraft-fossils-restart.timer b/ansible/roles/minecraft_fossils/files/minecraft-fossils-restart.timer new file mode 100644 index 0000000..aee390a --- /dev/null +++ b/ansible/roles/minecraft_fossils/files/minecraft-fossils-restart.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Restart minecraft-fossils daily at 08:00 Europe/Tallinn + +[Timer] +OnCalendar=*-*-* 08:00:00 Europe/Tallinn +Persistent=true +Unit=minecraft-fossils-restart.service + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/minecraft_fossils/handlers/main.yaml b/ansible/roles/minecraft_fossils/handlers/main.yaml new file mode 100644 index 0000000..c28484f --- /dev/null +++ b/ansible/roles/minecraft_fossils/handlers/main.yaml @@ -0,0 +1,4 @@ +--- +- name: Reload systemd + ansible.builtin.systemd: + daemon_reload: true diff --git a/ansible/roles/minecraft_fossils/tasks/main.yaml b/ansible/roles/minecraft_fossils/tasks/main.yaml new file mode 100644 index 0000000..1e20de5 --- /dev/null +++ b/ansible/roles/minecraft_fossils/tasks/main.yaml @@ -0,0 +1,49 @@ +--- +# Fabric server plus a sidecar that mirrors the whitelist, ops and ban +# lists from the Paper server. World data in /opt/minecraft-fossils/data +# is not managed here. A compose change recreates the container, so it +# is saved and stopped with time to finish first. +- name: Deploy Minecraft Fossils + ansible.builtin.include_role: + name: docker_service + vars: + service_name: "minecraft-fossils" + service_base_path: "{{ services_base_path }}/minecraft-fossils" + service_templates: + - src: "secrets.env.j2" + dest: "secrets.env" + owner: "{{ minecraft_fossils_owner }}" + group: "{{ minecraft_fossils_owner }}" + mode: "0600" + service_files: + - src: "docker-compose.yml" + dest: "docker-compose.yml" + owner: "{{ minecraft_fossils_owner }}" + group: "{{ minecraft_fossils_owner }}" + # secrets.env only matters at container start; never restart the + # game server as a side effect of a deploy + service_restart_on_change: false + service_graceful_stop: + - container: minecraft-fossils + timeout: 120 + pre_stop: docker exec minecraft-fossils rcon-cli save-all flush + health_check_enabled: false + +- name: Install the daily restart units + ansible.builtin.copy: + src: "{{ item }}" + dest: "/etc/systemd/system/{{ item }}" + mode: "0644" + loop: + - minecraft-fossils-restart.service + - minecraft-fossils-restart.timer + notify: Reload systemd + +- name: Flush handlers + ansible.builtin.meta: flush_handlers + +- name: Enable the daily restart timer + ansible.builtin.systemd: + name: minecraft-fossils-restart.timer + enabled: true + state: started diff --git a/ansible/roles/minecraft_fossils/templates/secrets.env.j2 b/ansible/roles/minecraft_fossils/templates/secrets.env.j2 new file mode 100644 index 0000000..616eec2 --- /dev/null +++ b/ansible/roles/minecraft_fossils/templates/secrets.env.j2 @@ -0,0 +1,2 @@ +RCON_PASSWORD={{ minecraft_fossils_rcon_password }} +CF_API_KEY={{ minecraft_fossils_cf_api_key }} diff --git a/ansible/roles/sillytavern/files/docker-compose.yml b/ansible/roles/sillytavern/files/docker-compose.yml new file mode 100644 index 0000000..b2453d6 --- /dev/null +++ b/ansible/roles/sillytavern/files/docker-compose.yml @@ -0,0 +1,11 @@ +services: + sillytavern: + image: ghcr.io/sillytavern/sillytavern:1.19.0 + container_name: sillytavern + restart: unless-stopped + ports: + - "10.20.0.20:8000:8000" + volumes: + - ./config:/home/node/app/config + - ./data:/home/node/app/data + - ./plugins:/home/node/app/plugins diff --git a/ansible/roles/sillytavern/tasks/main.yaml b/ansible/roles/sillytavern/tasks/main.yaml new file mode 100644 index 0000000..b616e77 --- /dev/null +++ b/ansible/roles/sillytavern/tasks/main.yaml @@ -0,0 +1,16 @@ +--- +# Reachable on the WireGuard address only. Bump the image tag in +# files/docker-compose.yml to upgrade. +- name: Deploy SillyTavern + ansible.builtin.include_role: + name: docker_service + vars: + service_name: "sillytavern" + service_base_path: "{{ services_base_path }}/sillytavern" + service_directories: + - { path: "config" } + - { path: "data" } + - { path: "plugins" } + service_files: + - { src: "docker-compose.yml", dest: "docker-compose.yml" } + health_check_enabled: false diff --git a/ansible/roles/traefik/tasks/main.yaml b/ansible/roles/traefik/tasks/main.yaml index 92e725c..2178bce 100644 --- a/ansible/roles/traefik/tasks/main.yaml +++ b/ansible/roles/traefik/tasks/main.yaml @@ -8,21 +8,25 @@ driver: bridge state: present -- name: Deploy Traefik using shared docker_service role +- name: Deploy Traefik ansible.builtin.include_role: name: docker_service vars: service_name: "traefik" service_base_path: "{{ traefik_base_path }}" service_directories: - - { path: "config", mode: "0755" } - - { path: "letsencrypt", mode: "0755" } - - { path: "logs", mode: "0755" } + - { path: "config" } + - { path: "letsencrypt" } + - { path: "logs" } service_templates: - - { src: "traefik.yml.j2", dest: "traefik.yml", mode: "0644" } - - { src: "dynamic.yml.j2", dest: "config/dynamic.yml", mode: "0644" } - - { src: "docker-compose.yml.j2", dest: "docker-compose.yml", mode: "0644" } - health_check_enabled: true - health_check_host: "{{ wireguard_ip }}" # Dashboard only accessible on WireGuard IP + - { src: "traefik.yml.j2", dest: "traefik.yml" } + - { src: "dynamic.yml.j2", dest: "config/dynamic.yml" } + - { src: "env.j2", dest: ".env" } + - { src: "docker-compose.yml.j2", dest: "docker-compose.yml" } + # The Coraza WAF middlewares were never wired to a router + service_absent: + - config/waf-middleware.yml + - config/waf-wp.yml + health_check_host: "{{ wireguard_ip }}" # dashboard listens on WireGuard only health_check_port: "{{ traefik_dashboard_port }}" health_check_delay: 3 diff --git a/ansible/roles/traefik/templates/docker-compose.yml.j2 b/ansible/roles/traefik/templates/docker-compose.yml.j2 index 33f485a..1f238d2 100644 --- a/ansible/roles/traefik/templates/docker-compose.yml.j2 +++ b/ansible/roles/traefik/templates/docker-compose.yml.j2 @@ -1,18 +1,16 @@ +name: traefik + services: traefik: - image: traefik:v3.2 container_name: traefik + image: traefik:{{ traefik_version }} restart: unless-stopped security_opt: - no-new-privileges:true - networks: - - {{ traefik_network }} ports: - "80:80" - "443:443" - - "{{ wireguard_ipv4 }}:8080:8080" # Dashboard only on WireGuard IP - environment: - - TZ=UTC + - "{{ wireguard_ip }}:{{ traefik_dashboard_port }}:8080" # Dashboard on WireGuard IP only volumes: - /etc/localtime:/etc/localtime:ro - /var/run/docker.sock:/var/run/docker.sock:ro @@ -20,6 +18,26 @@ services: - ./config:/config:ro - ./letsencrypt:/letsencrypt - ./logs:/var/log/traefik + networks: + - {{ traefik_network }} + env_file: + - .env + environment: + - TZ=UTC + labels: + - "traefik.enable=true" + - "traefik.docker.network={{ traefik_network }}" + + # Dashboard Router with IP Whitelist + - "traefik.http.routers.traefik-dashboard.rule=Host(`traefik.asxp.io`)" + - "traefik.http.routers.traefik-dashboard.entrypoints=websecure" + - "traefik.http.routers.traefik-dashboard.tls=true" + - "traefik.http.routers.traefik-dashboard.tls.certresolver=letsencrypt" + - "traefik.http.routers.traefik-dashboard.service=api@internal" + + # IP Whitelist Middleware (VPC + WireGuard) + - "traefik.http.middlewares.traefik-ipwhitelist.ipwhitelist.sourcerange=10.10.0.0/20,10.20.0.0/24" + - "traefik.http.routers.traefik-dashboard.middlewares=traefik-ipwhitelist" networks: {{ traefik_network }}: diff --git a/ansible/roles/traefik/templates/dynamic.yml.j2 b/ansible/roles/traefik/templates/dynamic.yml.j2 index c9ed6aa..1c0ccf2 100644 --- a/ansible/roles/traefik/templates/dynamic.yml.j2 +++ b/ansible/roles/traefik/templates/dynamic.yml.j2 @@ -1,15 +1,20 @@ +# Traefik Dynamic Configuration + http: middlewares: - # Security headers + # Security Headers Middleware secure-headers: headers: sslRedirect: true - forceSTSHeader: true + stsSeconds: 31536000 stsIncludeSubdomains: true stsPreload: true - stsSeconds: 31536000 + forceSTSHeader: true + frameDeny: true + contentTypeNosniff: true + browserXssFilter: true - # Rate limiting + # Rate Limiting Middleware rate-limit: rateLimit: average: 100 diff --git a/ansible/roles/traefik/templates/env.j2 b/ansible/roles/traefik/templates/env.j2 new file mode 100644 index 0000000..0c40b58 --- /dev/null +++ b/ansible/roles/traefik/templates/env.j2 @@ -0,0 +1,8 @@ +# Traefik Configuration + +# Let's Encrypt Email (required for SSL certificates) +LETSENCRYPT_EMAIL= + +# Timezone +TZ=UTC +LETSENCRYPT_EMAIL={{ letsencrypt_email }} diff --git a/ansible/roles/traefik/templates/traefik.yml.j2 b/ansible/roles/traefik/templates/traefik.yml.j2 index a873956..38244a7 100644 --- a/ansible/roles/traefik/templates/traefik.yml.j2 +++ b/ansible/roles/traefik/templates/traefik.yml.j2 @@ -1,7 +1,11 @@ +# Traefik Static Configuration + +# API and Dashboard api: dashboard: true - insecure: true # Dashboard on port 8080 (bound to WireGuard IP only) + insecure: true # Dashboard accessible on :8080 without auth (restricted by network) +# Entry Points entryPoints: web: address: ":80" @@ -10,10 +14,10 @@ entryPoints: entryPoint: to: websecure scheme: https - permanent: true websecure: address: ":443" +# Docker Provider providers: docker: endpoint: "unix:///var/run/docker.sock" @@ -23,6 +27,7 @@ providers: directory: /config watch: true +# Let's Encrypt Certificate Resolver certificatesResolvers: letsencrypt: acme: @@ -31,6 +36,7 @@ certificatesResolvers: httpChallenge: entryPoint: web +# Logging log: level: INFO filePath: /var/log/traefik/traefik.log @@ -38,3 +44,9 @@ log: accessLog: filePath: /var/log/traefik/access.log bufferingSize: 100 + fields: + headers: + defaultMode: drop + names: + User-Agent: keep + Referer: keep diff --git a/ansible/site.yaml b/ansible/site.yaml new file mode 100644 index 0000000..f1a81ca --- /dev/null +++ b/ansible/site.yaml @@ -0,0 +1,46 @@ +--- +# Everything the deploy pipeline manages. Idempotent: a run with nothing +# to change touches nothing and restarts nothing. +# +# ansible-playbook site.yaml --check --diff # dry run +# ansible-playbook site.yaml --tags traefik # one service +# +# Host bootstrap (WireGuard, SSH hardening, s3fs, swap) is in playbooks/ +# and is run by hand. + +- name: Host baseline + hosts: asxpio + become: true + roles: + - { role: base, tags: [base] } + +- name: asxpio-ops services + hosts: ops_servers + become: true + roles: + - { role: traefik, tags: [traefik] } + - { role: crawler_alert, tags: [crawler_alert] } + +- name: asxpio-services services + hosts: services_servers + become: true + roles: + - { role: sillytavern, tags: [sillytavern] } + - { role: minecraft, tags: [minecraft] } + - { role: minecraft_fossils, tags: [minecraft_fossils] } + +# Late: restarting Forgejo briefly cuts the runner off from its job log +- name: Forgejo + hosts: ops_servers + become: true + roles: + - { role: forgejo, tags: [forgejo] } + +# Last: when the pipeline changes the runner it recreates the container +# that is running the pipeline. Everything else is done by then; that job +# ends as failed and the next run confirms the result. +- name: Forgejo runner + hosts: services_servers + become: true + roles: + - { role: forgejo_runner, tags: [forgejo_runner] }