ansible: monitoring with Prometheus, Loki and Grafana

Nothing watched the servers: tonight's asxpio crash loop was found by hand. On asxpio-ops, Prometheus (30 days), Loki (14 days) and Grafana at gf.asxp.io (WireGuard only), with blackbox_exporter probing every service from outside and postgres_exporter. On both hosts node_exporter and cAdvisor on the VPC address, and Alloy shipping container logs, the journal and Traefik's access log to Loki over the VPC. Traefik and Forgejo expose their metrics; restic and the bucket replica record their last success for node_exporter. Dashboards: an overview as home, plus hosts, containers, Traefik, probes, Postgres and logs. Alerts by email to me@asxp.io: host or service down, certificate under 14 days, disk or memory nearly full, a container restart-looping, Postgres down, backup or replica stale. Assisted-by: Claude:opus-5-5
2026-11-13 04:59:11 +04:00 · 2026-11-13 04:59:11 +04:00 · e20f5a7d07
commit e20f5a7d07
parent 5289542dca
28 changed files with 24705 additions and 161 deletions
--- a/ansible/inventory/group_vars/all/monitoring.yaml
+++ b/ansible/inventory/group_vars/all/monitoring.yaml
@ -0,0 +1,34 @@
 ---
 # Monitoring: Prometheus, Loki and Grafana on asxpio-ops; node_exporter,
 # cAdvisor and Alloy on both hosts. Scrapes and log shipping use the VPC.
 # renovate: datasource=docker depName=prom/prometheus
 monitoring_prometheus_version: "v3.16.0"
 # renovate: datasource=docker depName=grafana/grafana
 monitoring_grafana_version: "12.4.15"
 # renovate: datasource=docker depName=grafana/loki
 monitoring_loki_version: "3.7.9"
 # renovate: datasource=docker depName=grafana/alloy
 monitoring_alloy_version: "v1.21.3"
 # renovate: datasource=docker depName=prom/node-exporter
 monitoring_node_exporter_version: "v1.12.1"
 # renovate: datasource=docker depName=ghcr.io/google/cadvisor
 monitoring_cadvisor_version: "v0.60.6"
 # renovate: datasource=docker depName=prom/blackbox-exporter
 monitoring_blackbox_version: "v0.29.1"
 # renovate: datasource=docker depName=quay.io/prometheuscommunity/postgres-exporter
 monitoring_postgres_exporter_version: "v0.20.1"
 monitoring_metrics_retention: 30d
 monitoring_logs_retention: 336h   # 14 days
 monitoring_alert_email: me@asxp.io
 # Agents listen on the host's VPC address only
 monitoring_node_exporter_port: 9100
 monitoring_cadvisor_port: 9180
 monitoring_loki_port: 3100
 monitoring_textfile_dir: /var/lib/node_exporter/textfile
 monitoring_grafana_host: gf.asxp.io
 monitoring_grafana_admin_password: "{{ vault_grafana_admin_password }}"
 monitoring_forgejo_metrics_token: "{{ vault_forgejo_metrics_token }}"
--- a/ansible/inventory/group_vars/all/vault.yaml
+++ b/ansible/inventory/group_vars/all/vault.yaml
@ -1,162 +1,170 @@
 $ANSIBLE_VAULT;1.1;AES256
-62383639653033613330393038663633626639623331333361623062323062646462353239616633
+62653531636435386335636364616132363938633234306136383934323061656135646462613361
-3832353736626233646462323464613639636462373835300a663964363162326364653838613165
+3565643532313232366136316437306231656365656238300a643861333935623734653466663563
-31613033663365663434353564653764366436643861363464323537623464386334313831353435
+36383330363030393064303035363639646362326132313433333232613933653532333965353163
-6639623166373233630a306431383263626264386663326163333031386235666263373631343965
+3939363632393635350a316436626464663234313137666337646635326535326165653831346565
-39653262303264333238323662313662626339393361346663353939323561353563623061643961
+32623865396532633862633632663363323466346232643335633733343237663337336262663564
-61613066383930303834373066373862616362396134626638343133326230663039303435656261
+34356162653338656533363965653264303931363461613431386363336364646137633537393338
-30336136343461613039663861633033626264633864383137613561376164353739336531643063
+64343637333462386365323535333962666566333561363033343062613330366466323063356237
-63376462383665653339623865336665663131313733623736313233653166356262316330303336
+63633662336364663164656536656237396630356364336534663931346636666435313935343436
-31376631376631383832306435343763653539643536326139396331393135666161623135383664
+65363533303162643435306232303131396161623837356163306531316462323739303866346263
-38356438306263633835313932613537356262353434373761626365386638643365633530393266
+36313535373539326132643033373161613738336133663439373866376530323163623061653163
-32643036323165623139633031333639333437383663653338643634333633643735313131646364
+62636362656362356636656338303064303832376165356566363164656633383430656330663439
-34643061376364636231363131636531336161623039343839376439663131613866653064316231
+65663332343161643466313261643734313735663635363836393735663533323939666632666637
-38666631663536636163316264383230613332663066373730643538613964393162313434353738
+66303932333438643436623539636636653139373562363733303637663165623366333237613261
-39663133313433323033633161633336643033383330353933643133303064616562666134313464
+39386333373462303439333963373264636565646635623532393365303635656136623362656561
-37386636386231303864303761383363616165343363326365616231353735613536623664316462
+33636635346261396661663662613336306662643635396261653636643639346466656130343764
-30626663333564336535633430383136646437366362333936633566666462653431643631383136
+31336337383763633139303430316366393931616138353637613765333337323764656230373731
-39393466363038643663633136643766356566323531623665666438646434353162343766653639
+34346632336133306433323535303162623763303135353834303766613839353939376436366464
-35366432326434333333343136613239346337326334343765633865376135396630383238373661
+62333263343561663836376134613736383131366638626265343462313164666461306464366430
-64386561366164333563363130646132393163356135626432356633646531656534343237666364
+39343266613535363465376133623437303966366533646663396136393135383136383536616633
-35613234303663396239316238646633663731323566343434373666336436383633373262616332
+34353864393961356533633163333035323164353061373366326266623734313631643637303936
-38396166373932643964383238376562666430633563623964376339636634373638646433303466
+61303935336537663865663431643739616430353933313766373466346435333263323630353961
-65623636653963323463366339653533303532323562343039623562646339646431333662633237
+39356335666364393239303937363134313231333264373139333865353139353738346561336138
-36656634373461393333353432306330623461333963303165373264313761633561323963396630
+36363265303161303536373736613265383232616466616635336232646336316265616531343932
-63656461646563343930376537633439646139396663336265343333383464653763303239333037
+62326133613764666536356262396163626437646230663933636534373062393831363238326461
-33313964326264313863356666626437363133646634616631623131363435316237646336323165
+63613639656535346561393234323063663765383234313339656332366539353866313739656533
-64333236343265303535306664636662313864633735306663386335363731303464386632326438
+38666232313330623738346561353165346664626439303263336138393763653432366162303837
-66663938393832613733626563623236613564643631626265643132316632376461613064323466
+61323733366161336439373564333937303137373334343132643137303935616361653036366538
-66383739613633386264343661353539333561313065316134666432656264363830636164313230
+64306434646665323866653664373539643233383962346137393366316639616637303732323432
-30323937313861396333623737643531306531376533303931363466393566666536386337396439
+35653065613033663763343933316631363764356137333661613433303139326135353863356435
-36373863363361333036633338303233643039653465363531666566336237616465306361636335
+33303730633262336433626232613334646365383533386164326232636139653639366362303830
-33323064623362653637353831373561653135323836356264666639393133633564366237356534
+63663632303263373038386230313130336235613033643565346435326466653838373366653530
-61653137313834633964393063346661346363333362633436363165316562316366346262366464
+61343661353235393262366439343563346232636431383938333130373534366538363637663931
-31656236333030373833386630356161383763623336633064633066326564316164666332366663
+33303938356666393738636231366264373061333765323339346362363337313234323631353331
-37333130353465326633643533626366373036313565663136323337633765633565636137343365
+37656134623861353866313936373430333063626664376162323232653635306361633162373061
-63333332366265376539653863633162316330393939333438653432393063346634373336306461
+39643132323162333961346261373132373962333137336332656437316332343364326434613336
-38633938346235313934633431393638363535393464313362663836373039373663393939323939
+33663961323661636439626337316565663835373938643034623534613263346432643435313438
-62653632313435643932336431623032663231313332623734396633383566653133656364313739
+33313561363866633964613635666335313031373465383666363937656133616665396666346163
-34383333636535353864656335383436326262393261663264346139373534373861613666616461
+36303932336262393862333935353932646337656331306438643432633165306265646535316536
-65626263363132313337666637626263313231633463383632363739316662666339363562383462
+36353530326263643765623562613433356439376238393531633235383461316462386139323864
-32363538353266646533643539643964376263313531626635643135643339353530333366363636
+31313838386232383861663738653937623866376637626364643730343830636161383135633338
-35663833376332306166613931353034373864396166323430383566633231346233373061343833
+61326635643765633631356262353661623865363839386166656530656239333261313961346661
-30653438323438313531376437303738646436623638663963643134663562663537366630363663
+61653663643963313764633931326530626433306466303839346663343637343565393732616235
-31383239643137363865633233343733643661356438366435363734643764613337323261353735
+65343639396537386435656263313337653636313135353465303635303331376263383565346261
-33666530323937376634303032373464663433386331336635303331653034353933313363303665
+64366366633834666539653062353166626138623231303437633563653238336561613234633835
-61363039643231623137366231623964623635396435313365393464343033313331333334303366
+65663262343637653532376238353837376561653732633738333933626561333932383732356132
-66333838353534626263303861343733313531663831623038653961376333303734386366373030
+30653066353063313836666563326165343933386336336233353066383338343339316534326161
-37346531623462363566376534313231386662356437643633383161373431396436626465366132
+31323438663433633139643061333635316633353262343234633034303965663361626438313830
-63383363326665366261343638353137363731383539623438303365663131316130383139636363
+62653238616233303466373034333262313932616565643330393363376337383662373631386338
-65616630303134666332666132303164383866303935316438383739363234306339323634666331
+39323030353564646330323361353737333936373137336362346635346665376165613336313431
-66656564343661643033306166656431303333616633303832626237396538366133623239373835
+30353463333335616433663563653261383534613730646230376533373735613263323966393839
-61383362646565346234333230396665393863376336646239343665353836336630626431356133
+66626364663266616332643633343165336131313665376132616339356639313761383761636463
-38313831633666613135643564383631643032383637623438393031653630313462333232396466
+31343565326139646663356431643634636538323933343136626434316162316463313464356539
-36353735306632363432343633313361633233663336623633633033323736386637326462343735
+31333638393337393835626331376131326331376638613739636336313536376363396331646334
-62303435663030393738393035353036343530636661303733393437323562356431343438363666
+66616136333964343436636563353837363731386662656466623239383865663836653563306632
-37623736363163663166336233353362306439346234656338313964623265323464633765303261
+63336365303731663465336539613637376332653165663830336133366661393530383836376538
-33346532653435376263653635393965323036656339303262303261373938316631333439653831
+62393032626633346238353663383761373637356361313163336138663537313937303766376137
-62366664303631356666386461653735386432363065633337626632303437373535656631356165
+31393130616539313162663236323565643636663764386238313532353064636163333832663933
-33356236623563353132613037376435396331323632306238643733656665393764363137663131
+64396439363135323866636164633437663639366262356565326133653335613731333230626532
-66313561333765613739343766646232353932623035333838336463393838643133313635353566
+35303965376238636463333634366630346161336638386238633032636335323838656139316138
-61373938613632376134336364626332623930646438353630353266333035353764363734636539
+36316530376433366632626462656332313334313235633138303235336666333666393362373839
-30376138663032356636353731383661623130393263643138373032343035623931316630383033
+64366438633463336538336230653739653537363635393839373235373737623861633434306636
-34336238363866396662333034616234366238386566616266326536653063316331303166306338
+61333364626135353037623237363838363834383431356661626139386331313563363130383834
-65303330623331383331393838613666393831303761393030653335663130316238633963366436
+33633933643361663433666665623266326233643361306435616236633139356138363462656533
-61333931646437376165666537373533636166376235303763393963386234376663366565353533
+64613130633135343631363639633066373663393336396665643532316233376531353261363538
-35303832383865623539616566373064626461633564613261643332663264396565623638386438
+30333563363435323133636432346333346139633361333738383438626338396263623961653163
-63383636356339303764333336303566353838653137623535343533393466363037353138303434
+36303731613062663064333538383261303561336536633231613939666134326338636633356239
-61646139373033333337396165353737313632316361323935316435616434313932616432383565
+31363739653430383965396461323830316237653839353065313632613530663737663230353531
-65643839363162306432663765643564393865626134613133653639333839643566303536306361
+35356464383037343739373435303662343261663637303365633064316632333634343933383733
-30373632363030636264326537623462313161373833633932333438383866353032383731363438
+37633164343932373962363338306535636138383834643232316531643339346564373136326361
-33396565393166323233323266363065613461613735373763306238363934623636393466323965
+61623638666237333266326364636133356234343263623637333039353131353536343439373836
-35396636666639336464353565623561643032326433313136633564313066346130353331303639
+38613764633038656362376637366335376561303735343663636232343262393263373731346461
-63313431633431323162366438343237353434343338343735313662626234333262343461633833
+32663966333932373264383935613632643633623966336233393736663339666135663037663936
-31663165643334396433626563633266633366383334313864663132323737333632383433333665
+36633366623234323137366561373233613738366431326338616661373632306338366634313737
-36623738336431383738656261653064313237386362643566656639396234303964653433313231
+33366564653933373332626162316335393837323664333530336635353139386363613338633432
-34656166383038653363353665333330613036663236373464346466636532643436623232326432
+34306435616539626239396163333963376336616336336338623461303261393265353930323765
-64626662636261633361663338363365323632646330396235663863613164323563616439646365
+32656262376237643862336238636264613663646338353835356131626264363736393030396331
-39373866633836653361343835663931386137383133323464333165396664313665333563343862
+63653864313637313462303764663364396532343838383833346466336162323939386461633733
-36356337313336313239613037343338323333653333363562653432613665333731663766353533
+62313335336132343236376539373863363134366563313563333733353237666334626432363362
-64323035386234323365633365336161366133613839373338636538623531633363383436323632
+65363865376561633437616365306332373564313038316635373538343232376165656639373932
-61643137646365383535333762393537316335376330396430663134343338393762653065366562
+37343735343863616161663861353338643533356638326230326437353535343432653632343935
-66333161376531386438396334333435303464313637643737343764346631333334363135376632
+32636239363434653139366131306237306262643364663638326333353331363130363464373964
-37636236363661393135306339333135633464643164386366323765363935336562393037613464
+66333138306233616164333838346533346561336334656166633838666164636438383939376434
-38326263663236373639653862356331323630396335656436323465303762666534373433303439
+62313031663131383335633666376137396162636466343033316565343337386664663135636236
-62386138646265613435366565396462396461336331616666306539623639616531623637353939
+33633934613037666632336237633632643230383433336263373864633765623339646661663334
-31346232306638373666303264363861353964366132356431313463653766363864396464343933
+66613765326639323832623835333361376562636530643636326266333333376263663732643763
-34353239376233656565356633386663346634393730376666303762386430633534646533366364
+65306133323534366133636330363830663834356161646436393866313130623835336533383530
-65656130383961613334666539313138386662303639376130343066633836663036623031306336
+34613539303164663064316234623339353635643939323361316361393761363031353036333137
-64356337386161366266396436663364643538333439626330616636353031363262666563323839
+33613630623362653533653961343139376164623332376431626464636436383762303733303138
-34336238303938343565646263623866663963323764363938333464313639323734356131633339
+30313132653530313532653632393065613435616666393264386266373037623936363962653234
-34346239656532396635303665386630326663383039353862646333306331313632336362383839
+64303461623839303035386533396266303534306431343864323130373732393430373062326637
-65626332393863386437376531396232353734353233356231666438373166353433396163383762
+66396133376666383238303761663061396238323963643134626165356331616332663264633838
-39626236373937313438323861383536353663656562366338626562313464643935633939646236
+35663765346633653064393237613439626465303530316130366665643131613864353331366331
-32323936636362333731666463306534383037323362656266366432333131646162313237336633
+65396533376438323036366137383736666638393061346565613833643239646363393064633666
-33386664316263656632653234306531383162376239383132356437323139393666323565373062
+66326261313932343039306465653861366136313933323134376236306539346336313039393136
-30346432343763646664313262636538633365316436663862646563393236353262613831366535
+65353036353038323764653065373033306638636538356138363364643135303263613564666665
-30333030376330303237343537666265643930613364366361303062343930346432376231386536
+61316166616230313039656433636232626630653665643963336632366637663163633933373237
-31346330383030376464623635343762316431356434663063323534643637646666626465616430
+66653733663638313038363366643866633865303839613033323161353030393330316133303130
-64346634313361363563616565306562353831353062646363666163386438363836356164633039
+30666634633761666438386364663762343332613466316338623634303263386333366266366633
-30626532376266376462666639303536353262666135613134616430393566373261613062386663
+32663536396539376661363164656537633330636637313833326131643463643434636663346431
-38326533306539663763373230303536363064633364666363336432323139393233643736363239
+30356333343266616632376264663264393462323736326138643737636264306339356137653436
-38643464643664626364666631363962323034623938303664623562396166323932303465623738
+38333062393439333631396463363131326665646230616535633731313564626435313832383430
-36613330376266383637616437653533316461653866356235363839656662656630303231393337
+31633231663536663437396462336231666363646230626533663265616435346366653532323835
-63343061376330376566313738366639646662336334323935666235363733653632323364646538
+33646131343335303632653631363637656538343032363631303565613738383533363237376233
-64383461326630303230393364303066666265376137643538303866623237623737636335646632
+64303538303934363137306639666433643338323930656465323533323532383763353938303662
-65353330386462353834346466643063653930386563636665626566623765613266316334633939
+61376138343032643762326631323164386635613838623539313931623934623363316435653864
-31653033623564323030343562356531343261376566663536343831666266663334326336313462
+35336539663661663363306164303965333534636663643765363831323537636665623633343639
-64663266633830373237323364366665373364383166343764306435636437653438366636313433
+39643664653465316334376437653932343934383536353533316338303261326162306338383531
-30653331636661343066326139646333633137326164643966363835343735643634363938646138
+32303434316362663331613131373833386338633138636563663666373165633134643230303839
-64313138633833353766393833386530353530633061343433323765666639656261643964636438
+66383033316431623735663833333631393031323363303164363238396539306333363039393737
-34303936663664366539373737386233363963613865633264313138623537356561313065646335
+38653230303062303437336234353765333636383937613062356338613534653862643561363966
-66376631353562376664613431383430626533323761643038363864613034393233383966623438
+31363438383530313032316136346264373632356238663033393837366665633433323338333937
-31383363376232343435666636663433393939313366343562353766396137633935613662626165
+63393937373434663961356661306132386464303632636135383333346439646166353737643132
-32663834663139663262663032663234616137353566643063636531613865313831363237323833
+66633336343930643266336630383838373436396130343633646530623066623630616634366362
-36363937373934613939663735346539643561303066623965623233346639623230666664363862
+62326537366365323839633161353238653934353739626565306263316266666336613235656336
-62323639636331656332333161363466666266303631613763653335383761643335373031663865
+31643533343961363736633865363961386536323935616133313965343739353963383264633065
-35636662343834663136666563343366633662663133316431643331333731386436343139616366
+61383532636330316134613531326231613132396663333864663866376439306464616162373931
-33656139626432303563393539363739313364373965653930616165643034313935646636323235
+63336438343463623236626563313230613030366437396162643636383163303465333661663234
-64383866343431613161363766336330323331323932313835636535393766336138353536303064
+38333832313563373838303565613434393232363030366439376333356638333735343035363138
-62663965376237623238656633313566363565353461613032373237356632663433393430366136
+30643032616465326166646263326339616237663265373963343438376463383336373637313430
-61313831316638343164653935376637383763616663663964613134323931383339626666306332
+31376338626164373136386431333633383731373736666439386664336132343136303138646265
-63613264323661356136613837333266396330366564633936323634626264306462393064623234
+30366235356666393030633735346139663432353837353236623638313266336637373964626266
-64306663623832343166366264313036653635356331383439643963363934623631303535323331
+37643766333736656664376264336635613038363830383935653331393164383436653938326266
-39613564336537363435386665393736653230333230353961363338363563383063376432396565
+65623461383462313763326464383466363531636234656536393633623938363330396461613366
-32616631613735303265666334326333633064366438343639623230326532663639393036613935
+65303363323764613863643763633832313632613538363436393834613466633266343239656164
-61653965323939343864623434383363363137333939323336306437356239343834366566343436
+31383833396437663135326436343264383963353265643737363336663166303764356566613536
-32353365613461646531336437326164323266393666623062326465646563326137373663616137
+63376463323637303464313132663539313333306665316538303539623535356335306133613065
-65356134323664323664356439306137373762373162643163633262663637303239366530636164
+31356438356234336261656131613731633338646165333632323931313334333131623335666331
-30306331653030646538336433383661633135306263613238376636656163366239346232643961
+65663934393239326563333039393337653032633437303638326463653436373839363662303065
-32653034376438306565363632613732316166316236383837646437636363333035356361646666
+32653337313235323334376639383336633861653137643833646234393438663561316431616664
-35343662396633353539316665306461313032393262366564303762376566396137653562353135
+35616331333239636230333166323262396666626139383031326634653230383561376362666463
-31633537343337643361613666623566643138373732363534613166356563356463616561326366
+35383066666531353237666464333535626165653136316461303966623530666564626130626264
-32323430633361393865343832326338373636343433313332353839356137376265306232326430
+33333035396536333438366138306463663531333135333530326565383964386533653630646232
-34616633303765613961646330653531306665333236383630643534386265663530363130366139
+66656662313933313063666164643134386536366630373536613531636332633663363539393761
-66343532643631326238343631613037376231663235633032616635653333396134666539343862
+36376464653435306665616639346132643634333365656237613439386132616332356434333737
-30383436326534373964656634326166356236323135333964366361646532383732373030643465
+35303137333366343639343666656665333134383335396333346365656632326534363330396235
-37653232333964613230663833303733353930636365633733333332313962393136666461613763
+34316130323662633165616162623530623866373134323964643134326663376234343165366265
-37366236326536313636303365383231653261666161313232353437623863333731373831373133
+34663662303938346132626434646364363835376439633764356335333734663365653637623064
-37343963373735346232383332303530343437646337623964666563323838653838393732396638
+61373465366464646239653539663266313662643933316465386132616534383333326635653639
-62383731333465633565356634643831323564346561613334303962663337633538613537633530
+62666363653031316137336431396338333866383433366664326663386533313839393365643039
-65303934656437376536623535396537616564316234366534326166666265633138626532333434
+62313239653532353439386161646438343732313735646637313533623862366336346664646534
-61316336383733616364653065326337663765383831663337623333336361346534343162616164
+64333237353137316662616434393339386137666563336236643038316564623136613733613630
-31343862303865303635623138373738396366653562313838366335623436386464323864633233
+37653763373236623033336363656166316463396230623963343435333465633863373962366138
-32663364303365613061343738623230313765393361653434373361306562333336346161643365
+65383166653063616435656236653939636435343362343332323866363134613231666537373237
-33303530616663633435326139356561333965633838353663633930336535346132346266643735
+35353838643535316639363130306266383736386130373965623131636665393666366663323964
-38303461363664366232356462313766313465333462653865633034636665396266643462333138
+38356338383166376531366265346235346233366632363133376465323666353563646639386131
-36373732656239666435383235323061306464313233373430373333663363383530386537353461
+37323363353765383163616133653665373965613361336466366431313336303337643131656132
-61396262633964616332343138343065356166633730363632326362656163363833643936633763
+62646463623731383031656564353934383236366264383131343939633235323265616234656638
-31386136376232366431633564316266646565363235376137636564303631636536646332343032
+31666232303263623236346531376364616361353635386131653235636566303432353866616163
-66616165353334333739336466646635383665383864373537643736653435373535386335326433
+32313561386234643533396263393739623366653239336161616238623638333061393165376530
-38653730666337643439306635303439623864346232656138643737373437333364363863363039
+61323637366636376166326661323262613739653663343930616338343735393030313433353035
-39333434306439343538646131636233316139306361613631366134636635623734333561336636
+31653233316263633666633132636231653231333833653230643164316130613563333637303333
-62323539636231623265333163353432346264383063323736366431303930323432363066393365
+34626130373933616437653132616632633663376138616164636362613764646563366437303066
-62623864376162316563613163356337643034323736663837643261636533303531623337663732
+35336561313666303034623662656361343861363763373236353063623162366336313538653736
-62636666336430666130663535336533363765643866613162653535633365656439356533393036
+62393933373231393531306334323861613932643362366265306133303432366631383630626461
-32383830663431356162356664376635663463333333313632343735613733333931333534316437
+30333665643836396162363032613031646234333965366532353862393534386638646138383366
-32396539346337396131623939346239356130376633613838326161313536333230663337373931
+32653465626566306366393565303663643134396561316163616664383236306430333839646466
-31353331366534363661626636663233376130383134626439623835653834303362326135313265
+32353031656631333464363038306466366266353439633331336466316362373432353430346430
-35363864346332336433656131386666313564623830336635623465356665333565633165363164
+36373836333031336334353838303133346461373032336537373432303332616363323464333830
-62303133663733363534336665323739656237623535386361373235653061663134363937333331
+61633334393630393061633532306365303736343737613733343436636636343237356230386232
-39326361636134306133643363333135343666663762616437653830386235376136656231393964
+37653333376239646532373564613364366364356435666136366163663531623136376337656537
-3561
+30376561633362373134663737616464383932646664383930323939353961646532363365623264
 33366330393430336134373462343334633563373561316132393632623131376363363366326462
 37653764643939336339626164333530396662656332366638306137313236303062396632623539
 62626134636362643231346434393838393632326464303534396332363065363431346236383439
 35633065663737656565303130656434396366323364313832323134383565633866626364393163
 39616337653064353035616666313439616138303866636235646637393965316562386263393833
 38646238343266383561626333356563613831353935303763343037663431636261373038623639
 39376563366635383565646231613961393338393837616631323836346431346163393166373735
 37306432393938363435303332336335393764306233633931373662356536616262
--- a/ansible/roles/forgejo/templates/app.ini.j2
+++ b/ansible/roles/forgejo/templates/app.ini.j2
@ -99,3 +99,8 @@ SHOW_FOOTER_VERSION = false
 SHOW_FOOTER_TEMPLATE_LOAD_TIME = false
 SHOW_FOOTER_POWERED_BY = false
 [metrics]
 ; scraped by Prometheus on asxpio-ops over the traefik network
 ENABLED = true
 TOKEN = {{ monitoring_forgejo_metrics_token }}
--- a/ansible/roles/monitoring/defaults/main.yaml
+++ b/ansible/roles/monitoring/defaults/main.yaml
@ -0,0 +1,24 @@
 ---
 monitoring_base_path: "{{ services_base_path }}/monitoring"
 # Hosts with node_exporter, cAdvisor and Alloy, scraped over the VPC
 monitoring_hosts:
  - { name: asxpio-ops, address: "{{ asxpio_ops_vpc_ip }}" }
  - { name: asxpio-services, address: "{{ asxpio_services_vpc_ip }}" }
 # Outside-in probes (blackbox_exporter). module: see blackbox.yml.j2
 monitoring_probes:
  - { name: asxp.io, target: "https://asxp.io/", module: http_2xx }
  - { name: blog, target: "https://blog.asxp.io/", module: http_2xx }
  - { name: czsk.it, target: "https://czsk.it/", module: http_2xx }
  - { name: datacenterbingo, target: "https://datacenterbingo.xyz/", module: http_2xx }
  - { name: datacenterbingo data.json, target: "https://datacenterbingo.xyz/data.json", module: http_2xx }
  - { name: forgejo, target: "https://git.asxp.io/", module: http_2xx }
  - { name: s3, target: "https://s3.asxp.io/", module: http_up }
  - { name: sillytavern, target: "http://{{ asxpio_services_wireguard_ip }}:8000/", module: http_up }
  - { name: xmpp c2s starttls, target: "xmpp.asxp.io:5222", module: xmpp_starttls }
  - { name: xmpp c2s tls, target: "xmpp.asxp.io:5223", module: tls_connect }
  - { name: xmpp s2s, target: "xmpp.asxp.io:5269", module: tcp_connect }
  - { name: turn, target: "xmpp.asxp.io:3478", module: tcp_connect }
  - { name: xray, target: "{{ hostvars['asxpio-services'].public_ip }}:443", module: tcp_connect }
  - { name: minecraft, target: "mc.asxp.io:25565", module: tcp_connect }
--- a/ansible/roles/monitoring/files/dashboards/Infrastructure/containers.json
+++ b/ansible/roles/monitoring/files/dashboards/Infrastructure/containers.json
@ -0,0 +1,781 @@
 {
 "annotations": {
  "list": [
   {
    "builtIn": 1,
    "datasource": "-- Grafana --",
    "enable": true,
    "hide": true,
    "iconColor": "rgba(0, 211, 255, 1)",
    "name": "Annotations & Alerts",
    "type": "dashboard"
   }
  ]
 },
 "editable": true,
 "gnetId": 14282,
 "graphTooltip": 0,
 "id": null,
 "iteration": 1617715580880,
 "links": [],
 "panels": [
  {
   "collapsed": false,
   "datasource": "prometheus",
   "gridPos": {
    "h": 1,
    "w": 24,
    "x": 0,
    "y": 0
   },
   "id": 8,
   "panels": [],
   "title": "CPU",
   "type": "row"
  },
  {
   "aliasColors": {},
   "bars": false,
   "dashLength": 10,
   "dashes": false,
   "datasource": "prometheus",
   "fieldConfig": {
    "defaults": {
     "custom": {}
    },
    "overrides": []
   },
   "fill": 1,
   "fillGradient": 0,
   "gridPos": {
    "h": 7,
    "w": 24,
    "x": 0,
    "y": 1
   },
   "hiddenSeries": false,
   "id": 15,
   "legend": {
    "alignAsTable": true,
    "avg": true,
    "current": false,
    "max": true,
    "min": false,
    "rightSide": true,
    "show": true,
    "total": false,
    "values": true
   },
   "lines": true,
   "linewidth": 1,
   "nullPointMode": "null as zero",
   "options": {
    "alertThreshold": true
   },
   "percentage": false,
   "pluginVersion": "7.4.5",
   "pointradius": 2,
   "points": false,
   "renderer": "flot",
   "seriesOverrides": [],
   "spaceLength": 10,
   "stack": true,
   "steppedLine": false,
   "targets": [
    {
     "expr": "sum(rate(container_cpu_usage_seconds_total{instance=~\"$host\",name=~\"$container\",name=~\".+\"}[5m])) by (name) *100",
     "hide": false,
     "interval": "",
     "legendFormat": "{{name}}",
     "refId": "A"
    }
   ],
   "thresholds": [],
   "timeFrom": null,
   "timeRegions": [],
   "timeShift": null,
   "title": "CPU Usage",
   "tooltip": {
    "shared": true,
    "sort": 0,
    "value_type": "individual"
   },
   "type": "graph",
   "xaxis": {
    "buckets": null,
    "mode": "time",
    "name": null,
    "show": true,
    "values": []
   },
   "yaxes": [
    {
     "$$hashKey": "object:606",
     "format": "percent",
     "label": null,
     "logBase": 1,
     "max": null,
     "min": null,
     "show": true
    },
    {
     "$$hashKey": "object:607",
     "format": "short",
     "label": null,
     "logBase": 1,
     "max": null,
     "min": null,
     "show": true
    }
   ],
   "yaxis": {
    "align": false,
    "alignLevel": null
   }
  },
  {
   "collapsed": false,
   "datasource": "prometheus",
   "gridPos": {
    "h": 1,
    "w": 24,
    "x": 0,
    "y": 8
   },
   "id": 11,
   "panels": [],
   "title": "Memory",
   "type": "row"
  },
  {
   "aliasColors": {},
   "bars": false,
   "dashLength": 10,
   "dashes": false,
   "datasource": "prometheus",
   "fieldConfig": {
    "defaults": {
     "custom": {}
    },
    "overrides": []
   },
   "fill": 1,
   "fillGradient": 0,
   "gridPos": {
    "h": 8,
    "w": 12,
    "x": 0,
    "y": 9
   },
   "hiddenSeries": false,
   "id": 9,
   "legend": {
    "alignAsTable": true,
    "avg": true,
    "current": false,
    "max": true,
    "min": false,
    "rightSide": true,
    "show": true,
    "total": false,
    "values": true
   },
   "lines": true,
   "linewidth": 1,
   "nullPointMode": "null as zero",
   "options": {
    "alertThreshold": true
   },
   "percentage": false,
   "pluginVersion": "7.4.5",
   "pointradius": 2,
   "points": false,
   "renderer": "flot",
   "seriesOverrides": [],
   "spaceLength": 10,
   "stack": true,
   "steppedLine": false,
   "targets": [
    {
     "expr": "sum(container_memory_rss{instance=~\"$host\",name=~\"$container\",name=~\".+\"}) by (name)",
     "hide": false,
     "interval": "",
     "legendFormat": "{{name}}",
     "refId": "A"
    }
   ],
   "thresholds": [],
   "timeFrom": null,
   "timeRegions": [],
   "timeShift": null,
   "title": "Memory Usage",
   "tooltip": {
    "shared": true,
    "sort": 0,
    "value_type": "individual"
   },
   "type": "graph",
   "xaxis": {
    "buckets": null,
    "mode": "time",
    "name": null,
    "show": true,
    "values": []
   },
   "yaxes": [
    {
     "$$hashKey": "object:606",
     "format": "bytes",
     "label": null,
     "logBase": 1,
     "max": null,
     "min": null,
     "show": true
    },
    {
     "$$hashKey": "object:607",
     "format": "short",
     "label": null,
     "logBase": 1,
     "max": null,
     "min": null,
     "show": true
    }
   ],
   "yaxis": {
    "align": false,
    "alignLevel": null
   }
  },
  {
   "aliasColors": {},
   "bars": false,
   "dashLength": 10,
   "dashes": false,
   "datasource": "prometheus",
   "fieldConfig": {
    "defaults": {
     "custom": {}
    },
    "overrides": []
   },
   "fill": 1,
   "fillGradient": 0,
   "gridPos": {
    "h": 8,
    "w": 12,
    "x": 12,
    "y": 9
   },
   "hiddenSeries": false,
   "id": 14,
   "legend": {
    "alignAsTable": true,
    "avg": true,
    "current": false,
    "max": true,
    "min": false,
    "rightSide": true,
    "show": true,
    "total": false,
    "values": true
   },
   "lines": true,
   "linewidth": 1,
   "nullPointMode": "null as zero",
   "options": {
    "alertThreshold": true
   },
   "percentage": false,
   "pluginVersion": "7.4.5",
   "pointradius": 2,
   "points": false,
   "renderer": "flot",
   "seriesOverrides": [],
   "spaceLength": 10,
   "stack": true,
   "steppedLine": false,
   "targets": [
    {
     "expr": "sum(container_memory_cache{instance=~\"$host\",name=~\"$container\",name=~\".+\"}) by (name)",
     "hide": false,
     "interval": "",
     "legendFormat": "{{name}}",
     "refId": "A"
    }
   ],
   "thresholds": [],
   "timeFrom": null,
   "timeRegions": [],
   "timeShift": null,
   "title": "Memory Cached",
   "tooltip": {
    "shared": true,
    "sort": 0,
    "value_type": "individual"
   },
   "type": "graph",
   "xaxis": {
    "buckets": null,
    "mode": "time",
    "name": null,
    "show": true,
    "values": []
   },
   "yaxes": [
    {
     "$$hashKey": "object:606",
     "format": "bytes",
     "label": null,
     "logBase": 1,
     "max": null,
     "min": null,
     "show": true
    },
    {
     "$$hashKey": "object:607",
     "format": "short",
     "label": null,
     "logBase": 1,
     "max": null,
     "min": null,
     "show": true
    }
   ],
   "yaxis": {
    "align": false,
    "alignLevel": null
   }
  },
  {
   "collapsed": false,
   "datasource": "prometheus",
   "gridPos": {
    "h": 1,
    "w": 24,
    "x": 0,
    "y": 17
   },
   "id": 2,
   "panels": [],
   "title": "Network",
   "type": "row"
  },
  {
   "aliasColors": {},
   "bars": false,
   "dashLength": 10,
   "dashes": false,
   "datasource": "prometheus",
   "fieldConfig": {
    "defaults": {
     "custom": {}
    },
    "overrides": []
   },
   "fill": 1,
   "fillGradient": 0,
   "gridPos": {
    "h": 8,
    "w": 12,
    "x": 0,
    "y": 18
   },
   "hiddenSeries": false,
   "id": 4,
   "legend": {
    "alignAsTable": true,
    "avg": true,
    "current": false,
    "hideEmpty": false,
    "hideZero": false,
    "max": true,
    "min": false,
    "rightSide": true,
    "show": true,
    "sideWidth": null,
    "total": false,
    "values": true
   },
   "lines": true,
   "linewidth": 1,
   "nullPointMode": "null",
   "options": {
    "alertThreshold": true
   },
   "percentage": false,
   "pluginVersion": "7.4.5",
   "pointradius": 2,
   "points": false,
   "renderer": "flot",
   "seriesOverrides": [],
   "spaceLength": 10,
   "stack": false,
   "steppedLine": false,
   "targets": [
    {
     "expr": "sum(rate(container_network_receive_bytes_total{instance=~\"$host\",name=~\"$container\",name=~\".+\"}[5m])) by (name)",
     "hide": false,
     "interval": "",
     "legendFormat": "{{name}}",
     "refId": "A"
    }
   ],
   "thresholds": [],
   "timeFrom": null,
   "timeRegions": [],
   "timeShift": null,
   "title": "Received Network Traffic",
   "tooltip": {
    "shared": true,
    "sort": 0,
    "value_type": "individual"
   },
   "type": "graph",
   "xaxis": {
    "buckets": null,
    "mode": "time",
    "name": null,
    "show": true,
    "values": []
   },
   "yaxes": [
    {
     "$$hashKey": "object:674",
     "format": "Bps",
     "label": null,
     "logBase": 1,
     "max": null,
     "min": null,
     "show": true
    },
    {
     "$$hashKey": "object:675",
     "format": "short",
     "label": null,
     "logBase": 1,
     "max": null,
     "min": null,
     "show": true
    }
   ],
   "yaxis": {
    "align": false,
    "alignLevel": null
   }
  },
  {
   "aliasColors": {},
   "bars": false,
   "dashLength": 10,
   "dashes": false,
   "datasource": "prometheus",
   "fieldConfig": {
    "defaults": {
     "custom": {}
    },
    "overrides": []
   },
   "fill": 1,
   "fillGradient": 0,
   "gridPos": {
    "h": 8,
    "w": 12,
    "x": 12,
    "y": 18
   },
   "hiddenSeries": false,
   "id": 6,
   "legend": {
    "alignAsTable": true,
    "avg": true,
    "current": false,
    "max": true,
    "min": false,
    "rightSide": true,
    "show": true,
    "total": false,
    "values": true
   },
   "lines": true,
   "linewidth": 1,
   "nullPointMode": "null",
   "options": {
    "alertThreshold": true
   },
   "percentage": false,
   "pluginVersion": "7.4.5",
   "pointradius": 2,
   "points": false,
   "renderer": "flot",
   "seriesOverrides": [],
   "spaceLength": 10,
   "stack": false,
   "steppedLine": false,
   "targets": [
    {
     "expr": "sum(rate(container_network_transmit_bytes_total{instance=~\"$host\",name=~\"$container\",name=~\".+\"}[5m])) by (name)",
     "interval": "",
     "legendFormat": "{{name}}",
     "refId": "A"
    }
   ],
   "thresholds": [],
   "timeFrom": null,
   "timeRegions": [],
   "timeShift": null,
   "title": "Sent Network Traffic",
   "tooltip": {
    "shared": true,
    "sort": 0,
    "value_type": "individual"
   },
   "type": "graph",
   "xaxis": {
    "buckets": null,
    "mode": "time",
    "name": null,
    "show": true,
    "values": []
   },
   "yaxes": [
    {
     "$$hashKey": "object:832",
     "format": "Bps",
     "label": null,
     "logBase": 1,
     "max": null,
     "min": null,
     "show": true
    },
    {
     "$$hashKey": "object:833",
     "format": "short",
     "label": null,
     "logBase": 1,
     "max": null,
     "min": null,
     "show": true
    }
   ],
   "yaxis": {
    "align": false,
    "alignLevel": null
   }
  },
  {
   "collapsed": false,
   "datasource": "prometheus",
   "gridPos": {
    "h": 1,
    "w": 24,
    "x": 0,
    "y": 26
   },
   "id": 19,
   "panels": [],
   "title": "Misc",
   "type": "row"
  },
  {
   "datasource": "prometheus",
   "fieldConfig": {
    "defaults": {
     "custom": {
      "align": null,
      "filterable": false
     },
     "mappings": [],
     "thresholds": {
      "mode": "absolute",
      "steps": [
       {
        "color": "green",
        "value": null
       },
       {
        "color": "red",
        "value": 80
       }
      ]
     }
    },
    "overrides": [
     {
      "matcher": {
       "id": "byName",
       "options": "id"
      },
      "properties": [
       {
        "id": "custom.width",
        "value": 260
       }
      ]
     },
     {
      "matcher": {
       "id": "byName",
       "options": "Running"
      },
      "properties": [
       {
        "id": "unit",
        "value": "d"
       },
       {
        "id": "decimals",
        "value": 1
       },
       {
        "id": "custom.displayMode",
        "value": "color-text"
       },
       {
        "id": "color",
        "value": {
         "fixedColor": "dark-green",
         "mode": "fixed"
        }
       }
      ]
     }
    ]
   },
   "gridPos": {
    "h": 10,
    "w": 24,
    "x": 0,
    "y": 27
   },
   "id": 17,
   "options": {
    "showHeader": true,
    "sortBy": []
   },
   "pluginVersion": "7.4.5",
   "targets": [
    {
     "expr": "(time() - container_start_time_seconds{instance=~\"$host\",name=~\"$container\",name=~\".+\"})/86400",
     "format": "table",
     "instant": true,
     "interval": "",
     "legendFormat": "{{name}}",
     "refId": "A"
    }
   ],
   "timeFrom": null,
   "timeShift": null,
   "title": "Containers Info",
   "transformations": [
    {
     "id": "filterFieldsByName",
     "options": {
      "include": {
       "names": [
        "container_label_com_docker_compose_project",
        "container_label_com_docker_compose_project_working_dir",
        "image",
        "instance",
        "name",
        "Value",
        "container_label_com_docker_compose_service"
       ]
      }
     }
    },
    {
     "id": "organize",
     "options": {
      "excludeByName": {},
      "indexByName": {},
      "renameByName": {
       "Value": "Running",
       "container_label_com_docker_compose_project": "Label",
       "container_label_com_docker_compose_project_working_dir": "Working dir",
       "container_label_com_docker_compose_service": "Service",
       "image": "Registry Image",
       "instance": "Instance",
       "name": "Name"
      }
     }
    }
   ],
   "type": "table"
  }
 ],
 "schemaVersion": 27,
 "style": "dark",
 "tags": [
  "cadvisor",
  "docker"
 ],
 "templating": {
  "list": [
   {
    "allValue": ".*",
    "current": {},
    "datasource": "prometheus",
    "definition": "label_values({__name__=~\"container.*\"},instance)",
    "description": null,
    "error": null,
    "hide": 0,
    "includeAll": true,
    "label": "Host",
    "multi": false,
    "name": "host",
    "options": [],
    "query": {
     "query": "label_values({__name__=~\"container.*\"},instance)",
     "refId": "Prometheus-host-Variable-Query"
    },
    "refresh": 1,
    "regex": "",
    "skipUrlSync": false,
    "sort": 5,
    "tagValuesQuery": "",
    "tags": [],
    "tagsQuery": "",
    "type": "query",
    "useTags": false
   },
   {
    "allValue": ".*",
    "current": {},
    "datasource": "prometheus",
    "definition": "label_values({__name__=~\"container.*\", instance=~\"$host\"},name)",
    "description": null,
    "error": null,
    "hide": 0,
    "includeAll": true,
    "label": "Container",
    "multi": false,
    "name": "container",
    "options": [],
    "query": {
     "query": "label_values({__name__=~\"container.*\", instance=~\"$host\"},name)",
     "refId": "Prometheus-container-Variable-Query"
    },
    "refresh": 1,
    "regex": "",
    "skipUrlSync": false,
    "sort": 0,
    "tagValuesQuery": "",
    "tags": [],
    "tagsQuery": "",
    "type": "query",
    "useTags": false
   }
  ]
 },
 "time": {
  "from": "now-6h",
  "to": "now"
 },
 "timepicker": {},
 "timezone": "",
 "title": "Cadvisor exporter",
 "uid": "gnet-14282",
 "version": 1,
 "description": "Simple exporter for cadvisor only"
 }
--- a/ansible/roles/monitoring/files/dashboards/Infrastructure/hosts.json
+++ b/ansible/roles/monitoring/files/dashboards/Infrastructure/hosts.json
--- a/ansible/roles/monitoring/files/dashboards/Logs/logs.json
+++ b/ansible/roles/monitoring/files/dashboards/Logs/logs.json
@ -0,0 +1,248 @@
 {
 "annotations": {
  "list": [
   {
    "$$hashKey": "object:75",
    "builtIn": 1,
    "datasource": "-- Grafana --",
    "enable": true,
    "hide": true,
    "iconColor": "rgba(0, 211, 255, 1)",
    "name": "Annotations & Alerts",
    "type": "dashboard"
   }
  ]
 },
 "description": "Log Viewer Dashboard for Loki",
 "editable": false,
 "gnetId": 13639,
 "graphTooltip": 0,
 "id": null,
 "iteration": 1608932746420,
 "links": [
  {
   "$$hashKey": "object:59",
   "icon": "bolt",
   "includeVars": true,
   "keepTime": true,
   "tags": [],
   "targetBlank": true,
   "title": "View In Explore",
   "type": "link",
   "url": "/explore?orgId=1&left=[\"now-1h\",\"now\",\"Loki\",{\"expr\":\"{job=\\\"$app\\\"}\"},{\"ui\":[true,true,true,\"none\"]}]"
  },
  {
   "$$hashKey": "object:61",
   "icon": "external link",
   "tags": [],
   "targetBlank": true,
   "title": "Learn LogQL",
   "type": "link",
   "url": "https://grafana.com/docs/loki/latest/logql/"
  }
 ],
 "panels": [
  {
   "aliasColors": {},
   "bars": true,
   "dashLength": 10,
   "dashes": false,
   "datasource": "loki",
   "fieldConfig": {
    "defaults": {
     "custom": {},
     "links": []
    },
    "overrides": []
   },
   "fill": 1,
   "fillGradient": 0,
   "gridPos": {
    "h": 3,
    "w": 24,
    "x": 0,
    "y": 0
   },
   "hiddenSeries": false,
   "id": 6,
   "legend": {
    "avg": false,
    "current": false,
    "max": false,
    "min": false,
    "show": false,
    "total": false,
    "values": false
   },
   "lines": false,
   "linewidth": 1,
   "nullPointMode": "null",
   "percentage": false,
   "pluginVersion": "7.1.0",
   "pointradius": 2,
   "points": false,
   "renderer": "flot",
   "seriesOverrides": [],
   "spaceLength": 10,
   "stack": false,
   "steppedLine": false,
   "targets": [
    {
     "expr": "sum(count_over_time({job=\"$app\"} |= \"$search\" [$__interval]))",
     "legendFormat": "",
     "refId": "A"
    }
   ],
   "thresholds": [],
   "timeFrom": null,
   "timeRegions": [],
   "timeShift": null,
   "title": "",
   "tooltip": {
    "shared": true,
    "sort": 0,
    "value_type": "individual"
   },
   "type": "graph",
   "xaxis": {
    "buckets": null,
    "mode": "time",
    "name": null,
    "show": true,
    "values": []
   },
   "yaxes": [
    {
     "$$hashKey": "object:168",
     "format": "short",
     "label": null,
     "logBase": 1,
     "max": null,
     "min": null,
     "show": false
    },
    {
     "$$hashKey": "object:169",
     "format": "short",
     "label": null,
     "logBase": 1,
     "max": null,
     "min": null,
     "show": false
    }
   ],
   "yaxis": {
    "align": false,
    "alignLevel": null
   }
  },
  {
   "datasource": "loki",
   "fieldConfig": {
    "defaults": {
     "custom": {}
    },
    "overrides": []
   },
   "gridPos": {
    "h": 25,
    "w": 24,
    "x": 0,
    "y": 3
   },
   "id": 2,
   "maxDataPoints": "",
   "options": {
    "showLabels": false,
    "showTime": true,
    "sortOrder": "Descending",
    "wrapLogMessage": false
   },
   "targets": [
    {
     "expr": "{job=\"$app\"} |= \"$search\" | logfmt",
     "hide": false,
     "legendFormat": "",
     "refId": "A"
    }
   ],
   "timeFrom": null,
   "timeShift": null,
   "title": "",
   "transparent": true,
   "type": "logs"
  }
 ],
 "refresh": false,
 "schemaVersion": 26,
 "style": "dark",
 "tags": [],
 "templating": {
  "list": [
   {
    "allValue": null,
    "current": {},
    "datasource": "loki",
    "definition": "label_values(job)",
    "hide": 0,
    "includeAll": false,
    "label": "App",
    "multi": false,
    "name": "app",
    "options": [],
    "query": "label_values(job)",
    "refresh": 1,
    "regex": "",
    "skipUrlSync": false,
    "sort": 0,
    "tagValuesQuery": "",
    "tags": [],
    "tagsQuery": "",
    "type": "query",
    "useTags": false
   },
   {
    "current": {
     "selected": false,
     "text": "",
     "value": ""
    },
    "hide": 0,
    "label": "String Match",
    "name": "search",
    "options": [
     {
      "selected": true,
      "text": "",
      "value": ""
     }
    ],
    "query": "",
    "skipUrlSync": false,
    "type": "textbox"
   }
  ]
 },
 "time": {
  "from": "now-1h",
  "to": "now"
 },
 "timepicker": {
  "hidden": false,
  "refresh_intervals": [
   "10s",
   "30s",
   "1m",
   "5m",
   "15m",
   "30m",
   "1h",
   "2h",
   "1d"
  ]
 },
 "timezone": "",
 "title": "Logs / App",
 "uid": "gnet-13639",
 "version": 13
 }
--- a/ansible/roles/monitoring/files/dashboards/Services/postgres.json
+++ b/ansible/roles/monitoring/files/dashboards/Services/postgres.json
--- a/ansible/roles/monitoring/files/dashboards/Services/probes.json
+++ b/ansible/roles/monitoring/files/dashboards/Services/probes.json
--- a/ansible/roles/monitoring/files/dashboards/Services/traefik.json
+++ b/ansible/roles/monitoring/files/dashboards/Services/traefik.json
--- a/ansible/roles/monitoring/files/dashboards/overview.json
+++ b/ansible/roles/monitoring/files/dashboards/overview.json
--- a/ansible/roles/monitoring/tasks/main.yaml
+++ b/ansible/roles/monitoring/tasks/main.yaml
@ -0,0 +1,37 @@
 ---
 # Prometheus, Loki, Grafana, blackbox_exporter and postgres_exporter on
 # asxpio-ops. Grafana: https://gf.asxp.io, WireGuard only.
 - name: Create the provisioning directories
  ansible.builtin.file:
    path: "{{ monitoring_base_path }}/{{ item }}"
    state: directory
    mode: "0755"
  loop:
    - grafana/provisioning/datasources
    - grafana/provisioning/dashboards
    - grafana/provisioning/alerting
    - grafana/dashboards
 - name: Deploy the dashboards
  ansible.builtin.copy:
    src: dashboards/
    dest: "{{ monitoring_base_path }}/grafana/dashboards/"
    mode: "0644"
 - name: Deploy the monitoring stack
  ansible.builtin.include_role:
    name: docker_service
  vars:
    service_name: monitoring
    service_base_path: "{{ monitoring_base_path }}"
    service_templates:
      - { src: docker-compose.yml.j2, dest: docker-compose.yml }
      - { src: prometheus.yml.j2, dest: prometheus.yml }
      - { src: blackbox.yml.j2, dest: blackbox.yml }
      - { src: loki.yml.j2, dest: loki.yml }
      - { src: grafana.env.j2, dest: grafana.env, mode: "0600" }
      - { src: datasources.yaml.j2, dest: grafana/provisioning/datasources/datasources.yaml }
      - { src: dashboards.yaml.j2, dest: grafana/provisioning/dashboards/dashboards.yaml }
      - { src: alerting.yaml.j2, dest: grafana/provisioning/alerting/alerting.yaml }
      - { src: alert-rules.yaml.j2, dest: grafana/provisioning/alerting/rules.yaml }
    health_check_url: "http://{{ asxpio_ops_vpc_ip }}:{{ monitoring_loki_port }}/ready"
--- a/ansible/roles/monitoring/templates/alert-rules.yaml.j2
+++ b/ansible/roles/monitoring/templates/alert-rules.yaml.j2
@ -0,0 +1,170 @@
 # Managed by Ansible (role monitoring). Generated; see the commit adding it.
 {% raw %}
 apiVersion: 1
 groups:
  - orgId: 1
    name: asxpio
    folder: Alerts
    interval: 1m
    rules:
      - uid: host-down
        title: "Host down"
        condition: C
        data:
          - refId: A
            relativeTimeRange: { from: 900, to: 0 }
            datasourceUid: prometheus
            model: { refId: A, expr: "1 - up{job=\"node\"}", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
          - refId: C
            datasourceUid: __expr__
            model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [0] } } ] }
        noDataState: OK
        execErrState: Error
        for: 2m
        labels: { severity: critical }
        annotations: { summary: "{{ $labels.host }} stopped answering (node_exporter over the VPC)" }
      - uid: target-down
        title: "Scrape target down"
        condition: C
        data:
          - refId: A
            relativeTimeRange: { from: 900, to: 0 }
            datasourceUid: prometheus
            model: { refId: A, expr: "up{job!~\"node|probe_.*\"} == 0", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
          - refId: C
            datasourceUid: __expr__
            model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [-1] } } ] }
        noDataState: OK
        execErrState: Error
        for: 5m
        labels: { severity: warning }
        annotations: { summary: "{{ $labels.job }} on {{ $labels.instance }} is not answering metrics scrapes" }
      - uid: probe-down
        title: "Service down (outside-in probe)"
        condition: C
        data:
          - refId: A
            relativeTimeRange: { from: 900, to: 0 }
            datasourceUid: prometheus
            model: { refId: A, expr: "1 - probe_success", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
          - refId: C
            datasourceUid: __expr__
            model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [0] } } ] }
        noDataState: OK
        execErrState: Error
        for: 2m
        labels: { severity: critical }
        annotations: { summary: "{{ $labels.probe }} ({{ $labels.instance }}) is not reachable" }
      - uid: cert-expiry
        title: "Certificate expires soon"
        condition: C
        data:
          - refId: A
            relativeTimeRange: { from: 900, to: 0 }
            datasourceUid: prometheus
            model: { refId: A, expr: "(probe_ssl_earliest_cert_expiry - time()) / 86400", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
          - refId: C
            datasourceUid: __expr__
            model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: lt, params: [14] } } ] }
        noDataState: OK
        execErrState: Error
        for: 1h
        labels: { severity: warning }
        annotations: { summary: "{{ $labels.probe }}: certificate expires in {{ $values.A.Value | printf \"%.0f\" }} days" }
      - uid: disk-full
        title: "Disk almost full"
        condition: C
        data:
          - refId: A
            relativeTimeRange: { from: 900, to: 0 }
            datasourceUid: prometheus
            model: { refId: A, expr: "100 * (1 - node_filesystem_avail_bytes{fstype!~\"tmpfs|overlay|squashfs|fuse.*\"} / node_filesystem_size_bytes{fstype!~\"tmpfs|overlay|squashfs|fuse.*\"})", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
          - refId: C
            datasourceUid: __expr__
            model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [85] } } ] }
        noDataState: OK
        execErrState: Error
        for: 10m
        labels: { severity: warning }
        annotations: { summary: "{{ $labels.host }} {{ $labels.mountpoint }} is {{ $values.A.Value | printf \"%.0f\" }}% full" }
      - uid: memory-low
        title: "Memory almost exhausted"
        condition: C
        data:
          - refId: A
            relativeTimeRange: { from: 900, to: 0 }
            datasourceUid: prometheus
            model: { refId: A, expr: "100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
          - refId: C
            datasourceUid: __expr__
            model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [92] } } ] }
        noDataState: OK
        execErrState: Error
        for: 10m
        labels: { severity: warning }
        annotations: { summary: "{{ $labels.host }} memory is {{ $values.A.Value | printf \"%.0f\" }}% used" }
      - uid: container-restarting
        title: "Container restart-looping"
        condition: C
        data:
          - refId: A
            relativeTimeRange: { from: 900, to: 0 }
            datasourceUid: prometheus
            model: { refId: A, expr: "changes(container_start_time_seconds{name!=\"\"}[15m])", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
          - refId: C
            datasourceUid: __expr__
            model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [2] } } ] }
        noDataState: OK
        execErrState: Error
        for: 0m
        labels: { severity: critical }
        annotations: { summary: "{{ $labels.name }} on {{ $labels.host }} restarted {{ $values.A.Value | printf \"%.0f\" }} times in 15 minutes" }
      - uid: postgres-down
        title: "Postgres down"
        condition: C
        data:
          - refId: A
            relativeTimeRange: { from: 900, to: 0 }
            datasourceUid: prometheus
            model: { refId: A, expr: "pg_up", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
          - refId: C
            datasourceUid: __expr__
            model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: lt, params: [1] } } ] }
        noDataState: OK
        execErrState: Error
        for: 1m
        labels: { severity: critical }
        annotations: { summary: "Postgres (storage) is not answering" }
      - uid: backup-stale
        title: "Backup stale"
        condition: C
        data:
          - refId: A
            relativeTimeRange: { from: 900, to: 0 }
            datasourceUid: prometheus
            model: { refId: A, expr: "(time() - restic_backup_last_success_timestamp_seconds) / 3600", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
          - refId: C
            datasourceUid: __expr__
            model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [26] } } ] }
        noDataState: OK
        execErrState: Error
        for: 0m
        labels: { severity: warning }
        annotations: { summary: "restic on {{ $labels.host }}: last good backup {{ $values.A.Value | printf \"%.0f\" }} hours ago" }
      - uid: replica-stale
        title: "Bucket replica stale"
        condition: C
        data:
          - refId: A
            relativeTimeRange: { from: 900, to: 0 }
            datasourceUid: prometheus
            model: { refId: A, expr: "(time() - s3_replicate_last_success_timestamp_seconds) / 3600", instant: true, intervalMs: 30000, maxDataPoints: 43200 }
          - refId: C
            datasourceUid: __expr__
            model: { refId: C, type: threshold, expression: A, conditions: [ { evaluator: { type: gt, params: [26] } } ] }
        noDataState: OK
        execErrState: Error
        for: 0m
        labels: { severity: warning }
        annotations: { summary: "hel1 -> fsn1 bucket replica: last good run {{ $values.A.Value | printf \"%.0f\" }} hours ago" }
 {% endraw %}
--- a/ansible/roles/monitoring/templates/alerting.yaml.j2
+++ b/ansible/roles/monitoring/templates/alerting.yaml.j2
@ -0,0 +1,17 @@
 apiVersion: 1
 contactPoints:
  - orgId: 1
    name: email
    receivers:
      - uid: email-me
        type: email
        settings:
          addresses: {{ monitoring_alert_email }}
          singleEmail: true
 policies:
  - orgId: 1
    receiver: email
    group_by: [alertname, host]
    group_wait: 1m
    group_interval: 5m
    repeat_interval: 4h
--- a/ansible/roles/monitoring/templates/blackbox.yml.j2
+++ b/ansible/roles/monitoring/templates/blackbox.yml.j2
@ -0,0 +1,38 @@
 # Managed by Ansible (role monitoring).
 modules:
  http_2xx:
    prober: http
    timeout: 10s
    http:
      preferred_ip_protocol: ip4
      follow_redirects: true
  # Up means "answers": 401/403/404 are fine (s3 root, allowlisted UIs)
  http_up:
    prober: http
    timeout: 10s
    http:
      preferred_ip_protocol: ip4
      valid_status_codes: [200, 301, 302, 401, 403, 404]
  tls_connect:
    prober: tcp
    timeout: 10s
    tcp:
      preferred_ip_protocol: ip4
      tls: true
  tcp_connect:
    prober: tcp
    timeout: 10s
    tcp:
      preferred_ip_protocol: ip4
  # XMPP client port: stream header, STARTTLS, then the certificate
  xmpp_starttls:
    prober: tcp
    timeout: 10s
    tcp:
      preferred_ip_protocol: ip4
      query_response:
        - send: "<?xml version='1.0'?><stream:stream to='asxp.io' xmlns='jabber:client' xmlns:stream='http://etherx.jabber.org/streams' version='1.0'>"
        - expect: "urn:ietf:params:xml:ns:xmpp-tls"
        - send: "<starttls xmlns='urn:ietf:params:xml:ns:xmpp-tls'/>"
        - expect: "<proceed"
        - starttls: true
--- a/ansible/roles/monitoring/templates/dashboards.yaml.j2
+++ b/ansible/roles/monitoring/templates/dashboards.yaml.j2
@ -0,0 +1,9 @@
 apiVersion: 1
 providers:
  - name: asxpio
    type: file
    disableDeletion: true
    allowUiUpdates: false
    options:
      path: /var/lib/grafana/dashboards
      foldersFromFilesStructure: true
--- a/ansible/roles/monitoring/templates/datasources.yaml.j2
+++ b/ansible/roles/monitoring/templates/datasources.yaml.j2
@ -0,0 +1,14 @@
 apiVersion: 1
 datasources:
  - name: Prometheus
    uid: prometheus
    type: prometheus
    access: proxy
    url: http://prometheus:9090
    isDefault: true
    jsonData: { timeInterval: 30s }
  - name: Loki
    uid: loki
    type: loki
    access: proxy
    url: http://loki:3100
--- a/ansible/roles/monitoring/templates/docker-compose.yml.j2
+++ b/ansible/roles/monitoring/templates/docker-compose.yml.j2
@ -0,0 +1,87 @@
 # Managed by Ansible (role monitoring).
 services:
  prometheus:
    image: prom/prometheus:{{ monitoring_prometheus_version }}
    container_name: prometheus
    command:
      - --config.file=/etc/prometheus/prometheus.yml
      - --storage.tsdb.path=/prometheus
      - --storage.tsdb.retention.time={{ monitoring_metrics_retention }}
      - --web.enable-lifecycle
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
      - prometheus-data:/prometheus
    networks: [monitoring, traefik, storage]
    restart: unless-stopped
    mem_limit: 640m
  loki:
    image: grafana/loki:{{ monitoring_loki_version }}
    container_name: loki
    command: ["-config.file=/etc/loki/loki.yml"]
    ports:
      # Alloy on both hosts pushes here over the VPC
      - "{{ asxpio_ops_vpc_ip }}:{{ monitoring_loki_port }}:3100"
    volumes:
      - ./loki.yml:/etc/loki/loki.yml:ro
      - loki-data:/loki
    networks: [monitoring]
    restart: unless-stopped
    mem_limit: 512m
  grafana:
    image: grafana/grafana:{{ monitoring_grafana_version }}
    container_name: grafana
    env_file: [grafana.env]
    volumes:
      - grafana-data:/var/lib/grafana
      - ./grafana/provisioning:/etc/grafana/provisioning:ro
      - ./grafana/dashboards:/var/lib/grafana/dashboards:ro
    networks: [monitoring, traefik]
    restart: unless-stopped
    mem_limit: 384m
    labels:
      - traefik.enable=true
      - traefik.docker.network=traefik
      - traefik.http.routers.grafana.rule=Host(`{{ monitoring_grafana_host }}`)
      - traefik.http.routers.grafana.entrypoints=websecure
      - traefik.http.routers.grafana.tls=true
      - traefik.http.routers.grafana.tls.certresolver=letsencrypt
      - traefik.http.routers.grafana.middlewares=grafana-wireguard-only
      - traefik.http.middlewares.grafana-wireguard-only.ipallowlist.sourcerange={{ wireguard_network_cidr }}
      - traefik.http.services.grafana.loadbalancer.server.port=3000
  blackbox:
    image: prom/blackbox-exporter:{{ monitoring_blackbox_version }}
    container_name: blackbox
    command: ["--config.file=/etc/blackbox/blackbox.yml"]
    volumes:
      - ./blackbox.yml:/etc/blackbox/blackbox.yml:ro
    networks: [monitoring]
    restart: unless-stopped
    mem_limit: 64m
  postgres-exporter:
    image: quay.io/prometheuscommunity/postgres-exporter:{{ monitoring_postgres_exporter_version }}
    container_name: postgres-exporter
    # The password comes from the storage stack's own .env (same host)
    env_file: [{{ services_base_path }}/storage/.env]
    environment:
      DATA_SOURCE_URI: storage-postgres:5432/postgres?sslmode=disable
      DATA_SOURCE_USER: postgres
    entrypoint: ["/bin/sh", "-c", "DATA_SOURCE_PASS=\"$$POSTGRES_PASSWORD\" exec /bin/postgres_exporter"]
    networks: [monitoring, storage]
    restart: unless-stopped
    mem_limit: 64m
 volumes:
  prometheus-data:
  loki-data:
  grafana-data:
 networks:
  monitoring:
  traefik:
    external: true
  storage:
    external: true
--- a/ansible/roles/monitoring/templates/grafana.env.j2
+++ b/ansible/roles/monitoring/templates/grafana.env.j2
@ -0,0 +1,17 @@
 GF_SERVER_ROOT_URL=https://{{ monitoring_grafana_host }}/
 GF_SECURITY_ADMIN_USER=admin
 GF_SECURITY_ADMIN_PASSWORD={{ monitoring_grafana_admin_password }}
 GF_SECURITY_COOKIE_SECURE=true
 GF_USERS_ALLOW_SIGN_UP=false
 GF_AUTH_ANONYMOUS_ENABLED=false
 GF_ANALYTICS_REPORTING_ENABLED=false
 GF_ANALYTICS_CHECK_FOR_UPDATES=false
 GF_NEWS_NEWS_FEED_ENABLED=false
 GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/var/lib/grafana/dashboards/overview.json
 GF_SMTP_ENABLED=true
 GF_SMTP_HOST=smtp.fastmail.com:587
 GF_SMTP_USER={{ vault_forgejo_smtp_user }}
 GF_SMTP_PASSWORD={{ vault_forgejo_smtp_password }}
 GF_SMTP_FROM_ADDRESS={{ vault_forgejo_smtp_user }}
 GF_SMTP_FROM_NAME=asxpio monitoring
 GF_SMTP_STARTTLS_POLICY=MandatoryStartTLS
--- a/ansible/roles/monitoring/templates/loki.yml.j2
+++ b/ansible/roles/monitoring/templates/loki.yml.j2
@ -0,0 +1,37 @@
 # Managed by Ansible (role monitoring).
 auth_enabled: false
 server:
  http_listen_port: 3100
  log_level: warn
 common:
  path_prefix: /loki
  replication_factor: 1
  ring:
    kvstore:
      store: inmemory
  storage:
    filesystem:
      chunks_directory: /loki/chunks
      rules_directory: /loki/rules
 schema_config:
  configs:
    - from: "2026-11-01"
      store: tsdb
      object_store: filesystem
      schema: v13
      index:
        prefix: index_
        period: 24h
 limits_config:
  retention_period: {{ monitoring_logs_retention }}
  reject_old_samples: true
  reject_old_samples_max_age: 168h
  ingestion_rate_mb: 8
  ingestion_burst_size_mb: 16
  volume_enabled: true
 compactor:
  working_directory: /loki/compactor
  retention_enabled: true
  delete_request_store: filesystem
 analytics:
  reporting_enabled: false
--- a/ansible/roles/monitoring/templates/prometheus.yml.j2
+++ b/ansible/roles/monitoring/templates/prometheus.yml.j2
@ -0,0 +1,75 @@
 # Managed by Ansible (role monitoring).
 global:
  scrape_interval: 30s
  evaluation_interval: 30s
 scrape_configs:
  - job_name: node
    static_configs:
 {% for h in monitoring_hosts %}
      - targets: ["{{ h.address }}:{{ monitoring_node_exporter_port }}"]
        labels: { host: {{ h.name }} }
 {% endfor %}
  - job_name: cadvisor
    scrape_interval: 60s
    static_configs:
 {% for h in monitoring_hosts %}
      - targets: ["{{ h.address }}:{{ monitoring_cadvisor_port }}"]
        labels: { host: {{ h.name }} }
 {% endfor %}
    metric_relabel_configs:
      # Short-lived CI job containers would bloat the series count
      - source_labels: [name]
        regex: FORGEJO-ACTIONS-TASK-.*
        action: drop
  - job_name: alloy
    static_configs:
 {% for h in monitoring_hosts %}
      - targets: ["{{ h.address }}:12345"]
        labels: { host: {{ h.name }} }
 {% endfor %}
  - job_name: traefik
    static_configs:
      - targets: ["traefik:8082"]
        labels: { host: asxpio-ops }
  - job_name: forgejo
    metrics_path: /metrics
    authorization:
      credentials: "{{ monitoring_forgejo_metrics_token }}"
    static_configs:
      - targets: ["forgejo:3000"]
        labels: { host: asxpio-ops }
  - job_name: postgres
    static_configs:
      - targets: ["postgres-exporter:9187"]
        labels: { host: asxpio-ops }
  - job_name: monitoring
    static_configs:
      - targets: ["localhost:9090", "loki:3100", "grafana:3000", "blackbox:9115"]
        labels: { host: asxpio-ops }
 {% for module in monitoring_probes | map(attribute='module') | unique %}
  - job_name: probe_{{ module }}
    metrics_path: /probe
    params:
      module: [{{ module }}]
    static_configs:
 {% for p in monitoring_probes if p.module == module %}
      - targets: ["{{ p.target }}"]
        labels: { probe: "{{ p.name }}" }
 {% endfor %}
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox:9115
 {% endfor %}
--- a/ansible/roles/monitoring_agent/tasks/main.yaml
+++ b/ansible/roles/monitoring_agent/tasks/main.yaml
@ -0,0 +1,22 @@
 ---
 # node_exporter, cAdvisor and Alloy on every host. Metrics listen on the
 # host's VPC address only; Alloy ships Docker logs and the journal to Loki
 # on asxpio-ops over the VPC.
 - name: Create the node_exporter textfile directory
  ansible.builtin.file:
    path: "{{ monitoring_textfile_dir }}"
    state: directory
    mode: "0755"
 - name: Deploy the monitoring agents
  ansible.builtin.include_role:
    name: docker_service
  vars:
    service_name: monitoring-agent
    service_base_path: "{{ services_base_path }}/monitoring-agent"
    service_templates:
      - { src: docker-compose.yml.j2, dest: docker-compose.yml }
      - { src: config.alloy.j2, dest: config.alloy }
    health_check_enabled: true
    health_check_host: "{{ vpc_ip }}"
    health_check_port: "{{ monitoring_node_exporter_port }}"
--- a/ansible/roles/monitoring_agent/templates/config.alloy.j2
+++ b/ansible/roles/monitoring_agent/templates/config.alloy.j2
@ -0,0 +1,78 @@
 // Managed by Ansible (role monitoring_agent): logs of {{ inventory_hostname }} to Loki.
 discovery.docker "containers" {
  host = "unix:///var/run/docker.sock"
 }
 discovery.relabel "containers" {
  targets = discovery.docker.containers.targets
  rule {
    source_labels = ["__meta_docker_container_name"]
    regex         = "/(.*)"
    target_label  = "container"
  }
  rule {
    source_labels = ["__meta_docker_container_label_com_docker_compose_project"]
    target_label  = "compose_project"
  }
  rule {
    source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
    target_label  = "compose_service"
  }
 }
 loki.source.docker "containers" {
  host          = "unix:///var/run/docker.sock"
  targets       = discovery.docker.containers.targets
  relabel_rules = discovery.relabel.containers.rules
  labels        = { job = "docker" }
  forward_to    = [loki.process.containers.receiver]
 }
 loki.process "containers" {
  // CI job containers are many, short-lived and noisy: the runner logs suffice
  stage.match {
    selector = "{container=~\"FORGEJO-ACTIONS-TASK-.*\"}"
    action   = "drop"
  }
  forward_to = [loki.write.default.receiver]
 }
 loki.relabel "journal" {
  forward_to = []
  rule {
    source_labels = ["__journal__systemd_unit"]
    target_label  = "unit"
  }
  rule {
    source_labels = ["__journal_priority_keyword"]
    target_label  = "level"
  }
 }
 loki.source.journal "journal" {
  max_age       = "12h"
  relabel_rules = loki.relabel.journal.rules
  labels        = { job = "journal" }
  forward_to    = [loki.write.default.receiver]
 }
 {% if inventory_hostname == 'asxpio-ops' %}
 local.file_match "traefik" {
  path_targets = [{ "__path__" = "/traefik-logs/access.log", job = "traefik-access" }]
 }
 loki.source.file "traefik" {
  targets    = local.file_match.traefik.targets
  forward_to = [loki.write.default.receiver]
 }
 {% endif %}
 loki.write "default" {
  endpoint {
    url = "http://{{ asxpio_ops_vpc_ip }}:{{ monitoring_loki_port }}/loki/api/v1/push"
  }
  external_labels = { host = "{{ inventory_hostname }}" }
 }
--- a/ansible/roles/monitoring_agent/templates/docker-compose.yml.j2
+++ b/ansible/roles/monitoring_agent/templates/docker-compose.yml.j2
@ -0,0 +1,66 @@
 # Managed by Ansible (role monitoring_agent).
 services:
  node-exporter:
    image: prom/node-exporter:{{ monitoring_node_exporter_version }}
    container_name: node-exporter
    command:
      - --path.rootfs=/host
      - --web.listen-address={{ vpc_ip }}:{{ monitoring_node_exporter_port }}
      - --collector.textfile.directory=/textfile
      - --collector.filesystem.mount-points-exclude=^/(dev|proc|run|sys|var/lib/docker/.+)($|/)
    network_mode: host
    pid: host
    volumes:
      - /:/host:ro,rslave
      - {{ monitoring_textfile_dir }}:/textfile:ro
    restart: unless-stopped
    mem_limit: 64m
  cadvisor:
    image: ghcr.io/google/cadvisor:{{ monitoring_cadvisor_version }}
    container_name: cadvisor
    command:
      - --listen_ip={{ vpc_ip }}
      - --port={{ monitoring_cadvisor_port }}
      - --docker_only=true
      - --housekeeping_interval=30s
      - --store_container_labels=false
      - --whitelisted_container_labels=com.docker.compose.project,com.docker.compose.service
      - --disable_metrics=percpu,sched,tcp,udp,hugetlb,referenced_memory,cpu_topology,resctrl,advtcp,process
    network_mode: host
    privileged: true
    devices:
      - /dev/kmsg
    volumes:
      - /:/rootfs:ro
      - /var/run:/var/run:ro
      - /sys:/sys:ro
      - /var/lib/docker/:/var/lib/docker:ro
      - /dev/disk/:/dev/disk:ro
    restart: unless-stopped
    mem_limit: 192m
  alloy:
    image: grafana/alloy:{{ monitoring_alloy_version }}
    container_name: alloy
    command:
      - run
      - --server.http.listen-addr={{ vpc_ip }}:12345
      - --storage.path=/var/lib/alloy/data
      - /etc/alloy/config.alloy
    network_mode: host
    volumes:
      - ./config.alloy:/etc/alloy/config.alloy:ro
      - /var/run/docker.sock:/var/run/docker.sock:ro
      - /var/log/journal:/var/log/journal:ro
      - /run/log/journal:/run/log/journal:ro
      - /etc/machine-id:/etc/machine-id:ro
 {% if inventory_hostname == 'asxpio-ops' %}
      - /opt/traefik/logs:/traefik-logs:ro
 {% endif %}
      - alloy-data:/var/lib/alloy/data
    restart: unless-stopped
    mem_limit: 256m
 volumes:
  alloy-data:
--- a/ansible/roles/restic_backup/templates/restic-backup.j2
+++ b/ansible/roles/restic_backup/templates/restic-backup.j2
@ -22,6 +22,12 @@ declare -A REPOS=(
 log() { echo "restic-backup: $*"; }
 # Last good backup per repository, for Prometheus (node_exporter textfile)
 metric() {
  local f={{ monitoring_textfile_dir }}/restic_$1.prom
  printf '# HELP restic_backup_last_success_timestamp_seconds Last successful restic backup.\n# TYPE restic_backup_last_success_timestamp_seconds gauge\nrestic_backup_last_success_timestamp_seconds{repo="%s"} %s\n' "$1" "$(date +%s)" > "$f.tmp" && mv "$f.tmp" "$f"
 }
 case "${1:-}" in
 init)
  for name in "${!REPOS[@]}"; do
@ -56,6 +62,7 @@ run)
    [ "$(date +%a)" = "{{ restic_prune_weekday }}" ] && prune="--prune"
    log "$name: forget $prune"
    restic forget --host {{ inventory_hostname }} {{ restic_keep }} $prune || failed=1
    metric "$name"
  done
  rm -rf "$STAGE"
--- a/ansible/roles/s3_replicate/templates/s3-replicate.j2
+++ b/ansible/roles/s3_replicate/templates/s3-replicate.j2
@ -16,4 +16,8 @@ rclone sync hel1:{{ p.src }} fsn1:{{ p.dst }} \
 {% endfor %}
  --stats-one-line -v || failed=1
 {% endfor %}
 if [ $failed = 0 ]; then
  f={{ monitoring_textfile_dir }}/s3_replicate.prom
  printf '# HELP s3_replicate_last_success_timestamp_seconds Last fully successful hel1 to fsn1 replica.\n# TYPE s3_replicate_last_success_timestamp_seconds gauge\ns3_replicate_last_success_timestamp_seconds %s\n' "$(date +%s)" > "$f.tmp" && mv "$f.tmp" "$f"
 fi
 exit $failed
--- a/ansible/roles/traefik/templates/traefik.yml.j2
+++ b/ansible/roles/traefik/templates/traefik.yml.j2
@ -16,6 +16,15 @@ entryPoints:
          scheme: https
  websecure:
    address: ":443"
  # Prometheus metrics, reached on the traefik network only (not published)
  metrics:
    address: ":8082"
 metrics:
  prometheus:
    entryPoint: metrics
    addRoutersLabels: true
    addServicesLabels: true
 # Docker Provider
 providers:
--- a/ansible/site.yaml
+++ b/ansible/site.yaml
@ -14,12 +14,19 @@
  roles:
    - { role: base, tags: [base] }
 - name: Monitoring agents
  hosts: asxpio
  become: true
  roles:
    - { role: monitoring_agent, tags: [monitoring_agent, monitoring] }
 - name: asxpio-ops services
  hosts: ops_servers
  become: true
  roles:
    - { role: traefik, tags: [traefik] }
    - { role: crawler_alert, tags: [crawler_alert] }
    - { role: monitoring, tags: [monitoring] }
 - name: asxpio-services services
  hosts: services_servers