ansible: nightly restic backups to both object storage regions

Hetzner's server backups cost 20% of each server's price and only
restore whole disks. Back up what matters with restic instead, into the
backup buckets we already pay for: one repository per host in hel1
(asxpio-backups-main) and one in fsn1 (asxpio-backups-repl), written
independently so either region alone is a full copy.

Postgres goes in as a dump, Forgejo's and vhpbot's SQLite databases
through the online backup API, Garage with a fresh metadata snapshot,
ejabberd through its own backup command. Keeps 7 daily, 4 weekly and
6 monthly snapshots, prunes on Sundays, and mails the journal when a
run fails.

restic is the pinned upstream release, so its version is tracked in
the repo like everything else. The vault gains the restic password
and the Xray config for the role that follows.

Assisted-by: Claude:opus-5-5

ansible/inventory/group_vars/all/vault.yaml

@@ -1,106 +1,154 @@
 $ANSIBLE_VAULT;1.1;AES256
-37373533663231633738353864666365663334666134346462306537373463616438663334613764
-6264643666356130396536323235333837316431373536370a623833383939373038366263393161
-62623563386130666637353164363331373061643563313861643239313363323234303738343738
-3864363533393962650a633633613138313066336238336363306161306430626165343039356664
-64643333633938306138636334656631316232626539336466313033373430623263376635626238
-33343839656639366363313535663561666535376461616366376232303034343064626334626432
-36663362386561616537343835373839363632363730386666653066343731643330623266353434
-39306664363561316138363862356663366365643738336536353137396463623630343430346137
-63336164393939666266306536363435663661663939333238373639346431636635323834356437
-31316632333731343362303331353338333830356635643431316165633963356264376134623665
-37626263336339386234376534366463646537623631303462366466633034343932346132363532
-65313237303966333864373664303931653833323036316434666338353430303333386338356262
-62633533613061653262323630366230303561623661303233316131373066343666303635323637
-63313739623230336461613530623732636262393239306663643835396365636166616536316336
-39376163633937353564343766326462663362396239383835363030393539393936383535353561
-64643231346465393261626266666532636265356531356463393839636265613431383335663263
-32353864333335333632653461366338386664373939623234306466323933316161633364353437
-31626164386430656432363861316464313336613466383562663838393738633337303139386264
-61363933653237653237353836313733666266666534333331376633643938323264623664663733
-38616565343830616638623032646533653433663935363563336665623638313637643337646439
-63666661366463313131363433643461626361303130653761333538303739653265646263626263
-37373137373038643365326566383237393236613831393833326630386237316133333935336234
-32353732633334393830306435313435363538626566386135326338363630666561623633323865
-62313732363562643766386462646531383238363962313437373735313262633162643666303136
-30393332663865313064653233653761333364306134646163373365376532393838666262366535
-38633135363932636533333139386136363964316538316435343633666637306139313033666531
-39396138346461333238623362343162663366333738616663353963653061663531343230326265
-62353466393564666462633462303939346532323931306533323132643862653265346137353738
-63366238323130656430313831326330623961383237346436633165366634356466636162646261
-30303962386532353739643464653139386336393737316266653764373738373530323863326233
-36383135366336333831383438323261636665666331663338613266666338623630363533366338
-62626231306135643830336235353338396336646238396633363561326163623132653638306131
-39356634313130373862356263393364623337363136626361396536616631326233343730643864
-63633633343662343833333064356432373235653864303863656163643830333339656265646639
-65623636333630643236316434326264356435393837626330383962336161633262356638393266
-66323337346434333563323032393563663831653437336634613665306331366662623530393263
-34313033303966633235323339356461356630636238313031633834383434343035353438383035
-32386462323232623861626530373663333564333932616563613432383134393661303264303130
-30353431353062373533303237653966373762346264613734643730663932643633353631666565
-33633337633561343733623434653862383364646434663333366334323739653933313537363366
-65396532346439653334653739376562343136346266333739396630373166613763313563396133
-30303530326465653339343035623333646238393034323862623765633139396364346339326233
-36656664303536623834326662666537343537643336366138383935343065313538613363373966
-35393961356531623262363163343034656130383235373937643639613331356333323964633966
-65343632303237653939393438636137316332313931353934313034326366623832353363636564
-66363437616663356431613063613535666336383730333337623439653130653439663830633534
-39356266346231323535646265646439326137346361656632323564313435626165643231653735
-62636530383961343662393232666664363034656534663762393262356163336337373862316331
-62346535636464633363663330323335386230633465613965613630613135366131313138653061
-36613930653430643236323162643864353863323264323731333037306433376236306138303032
-65333963333364326665616333353031323761653764316362336337386230633736363766333662
-64316662343265623064643031326534346639353566376661343462613537373862303233316538
-64323364366430326264366232666263396235363736366437346438656265386464323264653733
-63363263636233353761356331613330353535346336366437613837343061663362663031333964
-34396366313363383863303035633934376432393334663364313866396166623563646530663632
-64613336653635636564373632646337386438333431326264373766306564326430326237363964
-65386631393135313566656435353039363638393535613063393730316634313837623630323161
-36356331323165613537386164343831386633663936313537306639633262633239353830313530
-31363066316332373035656134333937373263633561616431323334336335656131383436626464
-64356634626636356331383962643735303335363863343561373736363134646536393431346432
-62373434323665343533636237363461303335663632326332633535303139393433623032613465
-66383137613531376333323431356566653462616330656532633361623338343637613239303433
-62626537373165666562646561373731383436353436323434653562396334313639616661643935
-30616431313936626637353966366531316463616366306335656664613366303665346263633830
-31303939623631636463366132613866343032633365326433356463643832353230653339646161
-34636637373464386162623961373933363863346637313535643134613265383166393634656131
-64343930643162613362386465306465333235663461303566366266303664316565396563333433
-61356130333833653330666633346663373634396634383030346138313938636134303666623962
-62356636633062646639323963653565386464373736626161633565303335336262383230363732
-39393539333262326234333537356461663033303933383237346235373734656536303862333236
-65643563343932333632386635346438323339343463323338383865343362333430346338633932
-32386139343866633139313533663832356633633663616261386364373065373365386334383932
-63663334616532316133353565643936643035346464643936643862333466363166333066326338
-30326462333463393834303836653437353033313736643365313739313334383030636535656434
-32373231366462643233363837373133366139366130393537363361346133653839396432343936
-34303030373638303934343463326365663466633963323033323066383734336331353864313564
-64393638353735646233326638346464336637373465663562336661363834343138376635323363
-65356537356137336563383866356665383132366136633035646363393933643165656236313863
-30386363393466636435643666663533623332656232393537656161613837663064353838653835
-66666164386566663536653833613062333432363966303066323563363664333464633038353561
-32626634643264373132323638643766386362613639303262633462663737663337393631303436
-62326438396235666338313033383835306566366365396331653633323335333865366565343036
-62313835333964366537323565653662323563383663323033623939346533656665303235313264
-32623163383530376233353738633062346530376161353664333166323538666465363237616166
-34633561636234623263663535333939306239303334363939616636343334383838643939636664
-63623331306332386634646533396232323063336166613339636533353337306531636135313465
-39636164323537643230353234303666313161633366633831356530633635636561373461643136
-66323734386239656635633365373734643038636566626666646434663061333530633837633166
-38663633393764346565316639323638623039326332646432666232656463626335313737623862
-66386439616635633061356461343736656138633064653733643737636361626366313733356665
-34346565613330663938333563623436323934353934393235356234613766633235316137376632
-32386333373235393837393035653538303433373733386562623037363837396431393531303839
-31633162386637333963633335383662623661336133653866653336376366363665623165633032
-65636261616664326661333930343862663766393035613164313837333561326330333930643166
-38393161316364306139376234643363346564613463613536613534373936653532353861633139
-39663130316130366333303834386162343336346664316439373839333465356564303062633434
-62626337616366303730663661383863653834623537333933616239386264326634656235363166
-63323061343639666135306437356665313466326339356230376234363964653631643561633236
-31386339343033353062383231666135303338356335366335663763383237626138633961393239
-66363539333161313138626332303662366336323266643062663762323561656634636539336265
-34653736306231356135626435653736366133386538643338346463653231653333306633666532
-61323266303764653734643262363564656233653135623234663232353762653766313565613736
-31613432363333366233323165616561646366643634376639343966373635343335343661636462
-32613563613139363932636330663764353164316231333461653965373635363337633662653962
-35346463643363376633356234626464666332646331643132643138346665393937
+32356366663237323730376532393462356237346236616131303336623262653236356331333330
+3035633139313236643830343031323136363663383338350a613731626164323234656637653630
+61613839303334303866613762346363373731333962323066666132313664616438373365613635
+3139633033346437390a643430646134376364393737343234386661353938623531353937666430
+65366364613362393262313863346432306265353264353464313335646365326233376631373732
+63666661363462663633633461323635633065613633636664383233353661643638316662386633
+61626635386463323230366565386163393063383436636666643037353337343339373035366539
+31353561386565383232306263333231326261633132303036666562326635313365346633636634
+32353739633631666134373963383865363935326130336336333739393130353831646632623562
+61373038376462363864333061626564623239396135626537366439373063313039313966363439
+39363837356434356166353839363035373233656665663838346563623433346333643038323130
+35333239313863303066366265373365636535313339616238343435613737356635643833633066
+39613835306532383930303130633535666238323864386466336131313637336361313137306161
+61653063663131666539363463326332383763643064653133343633633066366362373235663736
+65376239316364666161323661613661333265366432366137386538646333653166646538626364
+30636263336235336363646136663266376538646631323665636561313537393834336565313033
+65663339626638303365333438393261653432373438373065343862373339623361383530343232
+64333964643837333666663434356534653561313864383137303761346364633461653635656566
+32613434373365616132353061333864333063386335653638623165333765653230313939353037
+38633966323032623937376565383134383034376539383632393064666464313439633330666335
+34313038333366353433646236666265366235373439626162306435663364643333316562346439
+32653466613966303933633538366466636461373930353364323030646436316436383562646664
+62626230363830306366613530633363653265623531383936396539356132393634323434356437
+66636661626335323737643538343834303238346631666334393934343134366331346532396137
+39396662396437656138626230343438336637663163303735653335663134343862373862386465
+63643362643462663961663165306533623931313133636365616665333537643365623833303439
+30656338323062366263636666333636356565653338323366346431396434633261623833376635
+62346563373163663163643036383730663939613032663161396432653039373561663264646135
+36303836656563333431633565383130616335356539333934313065333638363739393930393232
+65366537623836356361393865396534376238663264643834333230636636666139643734336161
+63333164303132373136623834666566663864336331666533646130393465613039613166306135
+63373334346134396566656233393666333063313435626561356464363064643966373561346465
+30346334383766623539386338363363343938343166633834326261396637333963656437663137
+65633233383138666538663933303337623130613166373632623334386236313765633037613165
+65653565623264623965303133653938383261636163373238353632313365613264643834656330
+34383965666337663334333362623266356666643565626432383861366131336564353732356634
+62316164663463326539316264323433346430363431383131393832383530333634643065376536
+32616132626332623132333061353961396331366666326631303165303261326465303062323863
+34656164613132616138383537336466353461363233393161366566356664336331313661383835
+38393938633537313731626132376636343234336431393939643438363634326434333236333335
+36356538653465616663633337626439366132383434326539663434333133356364653133336639
+33313331373761643662313431386364303262383566333462323532343135376666333363333432
+61656234643666343531646665336534343533346530326265643335383962376662326631653638
+32626439303562623364353263316535646134363736323133393832653536353262306334393439
+32306531623134656438613738643562663339336438353537343837393164386338363263323762
+63333162316365323964313663383863363836643636326237626432623738323063303639373639
+65363330353736333831333431393332616264353263316338383165303237343037396362346265
+62663330643964663833346561363661613137646563396164383236633135343136653338353664
+35616164646332333338626336376138303133323335353031333566613732386335386365626635
+62353138343165346437666233633736646332313132396462633235323563636338333662636466
+37343364343533623034653165633263343238666265636365636162646262383834613238366635
+62656233653162346530393236343437623137313766306466646136643962343062643732306164
+38633432363365613864376436393566363239343761623838623232613033363438656136633231
+39303366623764616137646136353336343765353266363161343638303532356261666166616138
+64373865303035643866666335356639656134353339383666373332363631393030346433316466
+39663535393238613537343764316634633632616465343539643938313134653437386132336166
+32636365353161313030393830653565623933313165323661376635643638646237313661336566
+31323533353536306561663735383639353539663835366632333232343038646432666438326263
+35386162663264396562613338386336623536306530366266346336643763656532653733366534
+64393436313962356233323164383639363435656639303563393336623735613363373933313139
+37353930396531633531336333656339336264353864396461393162316365343261653939313039
+61653164646466613562386566663031356363383836633661373566386235626339646139393234
+61323237383131386531616463663465613731306636356230306530623261613838633562333031
+61326131393631633161386363386635633465613430616336356537616664383763333566363033
+35363634643537646337363639333039343761363032633934633339633765643330613035633939
+38353133646461653038383837656431306239306366323866623539383561316432626135386234
+63353533336330643263346338353538383663656161616162363538323061613237396332303866
+35326466663762616138373839383865316131346438353038626134313861613139336162616138
+33393665396439323738613030366439323136653164616163643433343438373131393666633638
+36333236323434376533626435353264663930663130343037616462306663616139363565313732
+32666166643830303536633439353130303235323166626634313139613565323235666131303432
+33316531623736333832356563383234346338663634366436306533366464366663646137396566
+31393034366439646161643361663436383437623839323366616537353436366366363036373565
+37396439633238313266383635623665396636333431313734303435323134356163656462323964
+39313335356133623234393536346539333231373035396233343832633566643337656237623465
+35623764343934646261333266613562363630396532653764303531383666363766383765363734
+37663537613633643830326237363335623566376639633065363330313462633566653832643866
+35633434303665656462653438313838636638333734376539383931396535333837393330333739
+34313434653366663737616331626136626634646333303637363563336466303938363630333334
+30656264633065333137643865633030643230636363396565383530653734653134303032376231
+30643834636361313131653266303633626565666234336638386538633739376238343831343035
+35373265343966666462633261323538633531653238346466616234313537323938316239383162
+35326438313065313534313534626535313738323035386163663666653339613166373837346532
+65623235353136323961623631636437656234633337653036353461313939373464383431343163
+39333836336462636562396161303636666238663432343838383833306130633639663262616361
+30323331386363373262356666303036343836343636353865613965643365396238313564333966
+36363031386637313230303536303232613331666334393535363365653733353965396436373561
+63636137313836346164383239363862353830333038363562396133343536633335633235333133
+32636263633635366664363235666330633364303531376166666530646336346664323338636262
+39643534353535323534366164633435343963333331353233633432623738313038326132363962
+63316563376163386139656566656661396664666231373433613337326238303362383563623661
+63353762336336303362383132643766663138313739366634666531616363616161613836666665
+31396265353033663166366137393030366265333232663161356636343863333633646166313466
+61636238613366623835383633346538306130626132346239346166653736343138613633373765
+39623437623531313138313731313934623837313039363233653930646336393934373361303834
+66373464323066303735663862383266346131323365313932356132633037366337373136313434
+39373832346264363633306535623733363666393032353031373336353239646461306162653534
+63323132626362316538323264316362656233346130326165333334333834643333636234626530
+35633531663034636637373762333632336262373066356530613631343664633764643765613131
+66353036356664646264336131323539633038373438623261313335663763386334336662376634
+35376661316333363539633239653337663064613033363931653362323633386664366138363137
+30336536343430333936656536663331656631613733396632343764363163313862373936323663
+36306432663539336531663738363365376431616262346664613134366664373536386433333566
+31346133643639613934383964656162623037373462633031353561623238636230633731323536
+31326562343339376664653065656635353662663634386263363330356566336661313536666265
+66666165356534306163326137336331306430336536663965323964656233396638636563356365
+31366161363366656463373063636433343039303037646533626633613362393639323736623437
+30386665326534623266633463643535356633303639346632666633363464306230653837333937
+32636538353431633963366132366331323838393639613535613237646466666532386535323966
+38323363353764313332336137653364343232323164353733646631343565316133356436343936
+35636132663964633135363335383062663464346330666164396330616435663663363664643064
+38306162616232633239313565383836356566376238303164393539666265313066346632303165
+62393866346363653831623063346631396631653863303038373239643964666238333461303734
+33343136333535616332633038646663623232663730376438386538633936323033356534356532
+30313638326633396261343937333464386135646231653262646162303739613232326139303034
+39326633383937313536636466373561393866303664653264353033363039613664656637656539
+32616230663138323964663236363763643833373161313066363434636334636337323035643131
+63636131663537313438316632376134313666393437303031356662626531653639303363346335
+36393862366433343365643738666132666130303331303837306432623731383863613230373130
+65623030366661653439303561356533336465656636356164646636373765336239353461636566
+35616335643430663666653363666431356436343439643464356635313764663335613335396161
+62393661353733336535633436626366653031323663383133383930383034313236353362376637
+35323438633761383366383066363663373563656465316134373162396130616463333634376431
+32323465356535653938303732663464653966613834313463616161343637306236393266656565
+64666465313665633064343639656532633331663362366434393566643630333732346465626237
+38373433623037396364663165623263326262323962616636613865393536626661356133613764
+66633036623335646166356537313337353634326239646262303964633836636463343735313966
+34386335393564613961623231336638313835613935316335306438343565333330303866646636
+31663731663762643136323062643433636666626330376364396230663765613466373336303730
+65326434393465336532353236366139633935666130653735346265643463366563306266666537
+34636662386238343032616263363736643435313839393038643934313263613635366537623666
+33623034386166663831616563343733653866326263396261636635363337663239366261623538
+31363132356166363664653466626238636565633265346631626632356337303631613432346435
+39666137653636393135306135313465313035656538336434393233663361326139643237626439
+39613461333766333939636461323430623137383434343563333734336236346336626337313736
+31396535633136623333336465366166376338376235623537366636636331653737346165656364
+62663831323530626133323138393231306534323166643430656237336537306231623131383839
+65306661326533326539646266363461373461646464366264393364333232653535333934623738
+66343266333530613032316633323662356339366635373166623833646130643337613436363537
+39353737303862666234323230303137623434633636303263653762393937613234383862313561
+39613138356237373639663565653664643265373133356436343332363961616438643937393735
+31653461353030626664393137343431663264636263326535646665373130313961316639636631
+33626533336362356631366133636666343236396263626133626132343762633163633339313663
+65643930383164353931646538653065616362636335333236353764643864613338623465633066
+31333135633963383361623533316636346434316137633235343134663266373234383664323539
+31306233303234336130346665356133326431306430366364636662663630663833303664356632
+36633263653435333261313065653634653736393837306633393936386561613838343736326331
+65613233343866313835313733613630363736613064366138326364636566343961663338376231
+30636538303030303038353637663732336136656263303737623664376165393861643831616464
+37643433316634336238333432643039643334366632326234646339356333663739633934323632
+36626230653437303762343039643235646432663831363338643466396439376537353964663464
+31353433363037363135316535643336653430383333633465376631653162373630323636636365
+35376531633239393636396363626432306565656432653938396535663739386239

ansible/inventory/host_vars/asxpio-ops.yaml

@@ -10,3 +10,22 @@ ssh_git_user: git
 # s3fs is off here: its ensure_diskfree reserve never fit the 38 GB disk
 s3fs_enabled: false
 s3fs_ensure_diskfree: 20000
+
+# restic: Postgres as a dump, Garage with a fresh metadata snapshot,
+# Forgejo's SQLite through the online backup API
+restic_paths:
+  - /etc
+  - /opt
+  - /root/backups/rclone-hetzner.sh
+  - /var/lib/docker/volumes/storage_garage_meta
+  - /var/lib/docker/volumes/storage_garage_data
+restic_excludes:
+  - /opt/backups
+  - /opt/containerd
+restic_pre_hooks:
+  - docker exec storage-postgres pg_dumpall -U postgres > "$STAGE/pg_dumpall.sql"
+  - docker exec storage-postgres pg_dump -U postgres -Fc asxpio > "$STAGE/asxpio.dump"
+  - docker exec storage-garage /garage meta snapshot --all
+  - sqlite3 /opt/forgejo/data/gitea/gitea.db ".backup '$STAGE/forgejo-gitea.db'"
+  - sqlite3 /opt/vhpbot-prod/bot.db ".backup '$STAGE/vhpbot-prod.db'"
+restic_schedule: "*-*-* 03:30:00"

ansible/inventory/host_vars/asxpio-services.yaml

@@ -6,3 +6,21 @@ wireguard_ip: "{{ asxpio_services_wireguard_ip }}"
 ssh_allow_public_git: false
 
 s3fs_ensure_diskfree: 81920  # 80 GB free = max 80 GB cache
+
+# restic: ejabberd's Mnesia through its own backup command, its SQLite
+# through the online backup API. The Paper world is copied live (RCON is
+# off, so there is no save-off); its autosave keeps that close enough.
+restic_paths:
+  - /etc
+  - /opt
+  - /var/lib/docker/volumes/xmpp_database
+  - /var/lib/docker/volumes/xmpp_tackygram
+restic_excludes:
+  - /opt/containerd
+  - /opt/minecraft/cache
+  - /opt/minecraft/libraries
+  - /opt/minecraft/versions
+restic_pre_hooks:
+  - docker exec xmpp-ejabberd-1 ejabberdctl backup /opt/ejabberd/database/nightly.backup
+  - sqlite3 /var/lib/docker/volumes/xmpp_database/_data/ejabberd.sqlite ".backup '$STAGE/ejabberd.sqlite'"
+restic_schedule: "*-*-* 04:15:00"

ansible/roles/restic_backup/defaults/main.yaml

@@ -0,0 +1,36 @@
+---
+# Nightly restic backup of the host to Hetzner Object Storage, one
+# repository per region, written independently: losing one region or one
+# repository leaves the other complete.
+
+# renovate: datasource=github-releases depName=restic/restic
+restic_version: "0.19.1"
+
+restic_password: "{{ vault_restic_password }}"
+restic_s3_access_key: "{{ s3_access_key }}"
+restic_s3_secret_key: "{{ s3_secret_key }}"
+
+restic_repositories:
+  hel1: "s3:https://hel1.your-objectstorage.com/{{ s3_bucket_backups_main }}/restic/{{ inventory_hostname }}"
+  fsn1: "s3:https://fsn1.your-objectstorage.com/{{ s3_bucket_backups_repl }}/restic/{{ inventory_hostname }}"
+
+# Dumps and snapshots taken by the hooks land here and are backed up too
+restic_stage_dir: /var/backups/restic-stage
+
+restic_paths: [/etc, /opt]
+restic_excludes: []
+# Shell lines run in order before the backup, with $STAGE set. A failing
+# hook marks the run as failed but the backup still runs.
+restic_pre_hooks: []
+
+restic_keep: "--keep-daily 7 --keep-weekly 4 --keep-monthly 6"
+restic_schedule: "*-*-* 03:30:00"
+# Prune (rewrites pack files, so costs S3 requests) once a week only
+restic_prune_weekday: Sun
+
+# Failure mail, through Forgejo's SMTP account
+restic_alert_smtp_host: smtp.fastmail.com
+restic_alert_smtp_port: 587
+restic_alert_smtp_user: "{{ vault_forgejo_smtp_user }}"
+restic_alert_smtp_password: "{{ vault_forgejo_smtp_password }}"
+restic_alert_recipient: "{{ vault_crawler_recipient }}"

ansible/roles/restic_backup/files/restic-alert

@@ -0,0 +1,20 @@
+#!/usr/bin/python3
+# Mail the tail of a failed unit's journal. Usage: restic-alert <unit>
+import os, smtplib, socket, subprocess, sys
+from email.message import EmailMessage
+
+unit = sys.argv[1]
+env = dict(l.rstrip("\n").split("=", 1) for l in open("/etc/restic/env") if "=" in l)
+log = subprocess.run(["journalctl", "-u", unit, "-n", "60", "--no-pager", "-o", "cat"],
+                     capture_output=True, text=True).stdout
+
+msg = EmailMessage()
+msg["Subject"] = f"[{socket.gethostname()}] {unit} failed"
+msg["From"] = env["ALERT_SMTP_USER"]
+msg["To"] = env["ALERT_RECIPIENT"]
+msg.set_content(log)
+
+with smtplib.SMTP(env["ALERT_SMTP_HOST"], int(env["ALERT_SMTP_PORT"]), timeout=30) as s:
+    s.starttls()
+    s.login(env["ALERT_SMTP_USER"], env["ALERT_SMTP_PASSWORD"])
+    s.send_message(msg)

ansible/roles/restic_backup/files/restic-alert@.service

@@ -0,0 +1,6 @@
+[Unit]
+Description=Mail the failure of %i
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/restic-alert %i

ansible/roles/restic_backup/handlers/main.yaml

@@ -0,0 +1,4 @@
+---
+- name: Reload systemd
+  ansible.builtin.systemd:
+    daemon_reload: true

ansible/roles/restic_backup/tasks/main.yaml

@@ -0,0 +1,88 @@
+---
+- name: Install sqlite3 for consistent copies of live databases
+  ansible.builtin.apt:
+    name: [sqlite3, bzip2]
+    state: present
+
+- name: Check the installed restic version
+  ansible.builtin.command: /usr/local/bin/restic version
+  register: restic_installed
+  changed_when: false
+  failed_when: false
+  check_mode: false
+
+- name: Install restic {{ restic_version }}
+  when: ("restic " ~ restic_version ~ " ") not in (restic_installed.stdout | default(''))
+  block:
+    - name: Download the release
+      ansible.builtin.get_url:
+        url: "https://github.com/restic/restic/releases/download/v{{ restic_version }}/restic_{{ restic_version }}_linux_amd64.bz2"
+        dest: "/tmp/restic_{{ restic_version }}.bz2"
+        checksum: "sha256:https://github.com/restic/restic/releases/download/v{{ restic_version }}/SHA256SUMS"
+        mode: "0600"
+
+    - name: Unpack it
+      ansible.builtin.shell: >
+        bunzip2 -c /tmp/restic_{{ restic_version }}.bz2 > /usr/local/bin/restic.new &&
+        chmod 755 /usr/local/bin/restic.new &&
+        mv /usr/local/bin/restic.new /usr/local/bin/restic &&
+        rm /tmp/restic_{{ restic_version }}.bz2
+      changed_when: true
+
+- name: Create config and cache directories
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    mode: "0700"
+  loop: [/etc/restic, /var/cache/restic]
+
+- name: Write credentials
+  ansible.builtin.template:
+    src: env.j2
+    dest: /etc/restic/env
+    mode: "0600"
+  diff: false
+
+- name: Install the scripts
+  ansible.builtin.template:
+    src: restic-backup.j2
+    dest: /usr/local/sbin/restic-backup
+    mode: "0700"
+
+- name: Install the alert script
+  ansible.builtin.copy:
+    src: restic-alert
+    dest: /usr/local/sbin/restic-alert
+    mode: "0700"
+
+- name: Install the systemd units
+  ansible.builtin.template:
+    src: "{{ item }}.j2"
+    dest: "/etc/systemd/system/{{ item }}"
+    mode: "0644"
+  loop: [restic-backup.service, restic-backup.timer]
+  notify: Reload systemd
+
+- name: Install the alert unit
+  ansible.builtin.copy:
+    src: restic-alert@.service
+    dest: /etc/systemd/system/restic-alert@.service
+    mode: "0644"
+  notify: Reload systemd
+
+- name: Flush handlers
+  ansible.builtin.meta: flush_handlers
+
+- name: Create the repositories
+  ansible.builtin.command: /usr/local/sbin/restic-backup init
+  register: restic_init
+  changed_when: "'created' in restic_init.stdout"
+  when: not ansible_check_mode
+
+- name: Enable the nightly timer
+  ansible.builtin.systemd:
+    name: restic-backup.timer
+    enabled: true
+    state: started
+  # On a first dry run the unit is not installed yet
+  ignore_errors: "{{ ansible_check_mode }}"

ansible/roles/restic_backup/templates/env.j2

@@ -0,0 +1,8 @@
+RESTIC_PASSWORD={{ restic_password }}
+AWS_ACCESS_KEY_ID={{ restic_s3_access_key }}
+AWS_SECRET_ACCESS_KEY={{ restic_s3_secret_key }}
+ALERT_SMTP_HOST={{ restic_alert_smtp_host }}
+ALERT_SMTP_PORT={{ restic_alert_smtp_port }}
+ALERT_SMTP_USER={{ restic_alert_smtp_user }}
+ALERT_SMTP_PASSWORD={{ restic_alert_smtp_password }}
+ALERT_RECIPIENT={{ restic_alert_recipient }}

ansible/roles/restic_backup/templates/restic-backup.j2

@@ -0,0 +1,70 @@
+#!/bin/bash
+# Managed by Ansible (role restic_backup).
+#
+#   restic-backup run                 hooks, then backup + forget to every repository
+#   restic-backup init                create the repositories that do not exist yet
+#   restic-backup <repo> <args...>    plain restic against one repository, e.g.
+#                                     restic-backup hel1 snapshots
+#                                     restic-backup fsn1 restore latest --target /tmp/r
+#
+# Repositories: {{ restic_repositories.keys() | join(', ') }}
+set -uo pipefail
+
+set -a; . /etc/restic/env; set +a
+export RESTIC_CACHE_DIR=/var/cache/restic
+STAGE={{ restic_stage_dir }}
+
+declare -A REPOS=(
+{% for name, url in restic_repositories.items() %}
+  [{{ name }}]="{{ url }}"
+{% endfor %}
+)
+
+log() { echo "restic-backup: $*"; }
+
+case "${1:-}" in
+init)
+  for name in "${!REPOS[@]}"; do
+    if RESTIC_REPOSITORY="${REPOS[$name]}" restic cat config >/dev/null 2>&1; then
+      log "$name: exists"
+    else
+      RESTIC_REPOSITORY="${REPOS[$name]}" restic init || exit 1
+      log "$name: created"
+    fi
+  done
+  ;;
+
+run)
+  failed=0
+  rm -rf "$STAGE"; install -d -m 700 "$STAGE"
+  export STAGE
+{% for hook in restic_pre_hooks %}
+  log "hook {{ loop.index }}/{{ restic_pre_hooks | length }}"
+  ( set -e; {{ hook }} ) || { log "hook failed"; failed=1; }
+{% endfor %}
+
+  for name in "${!REPOS[@]}"; do
+    export RESTIC_REPOSITORY="${REPOS[$name]}"
+    log "$name: backup"
+    restic backup --host {{ inventory_hostname }} --one-file-system --exclude-caches \
+{% for e in restic_excludes %}
+      --exclude '{{ e }}' \
+{% endfor %}
+      {{ (restic_paths + [restic_stage_dir]) | join(' ') }} || { failed=1; continue; }
+
+    prune=""
+    [ "$(date +%a)" = "{{ restic_prune_weekday }}" ] && prune="--prune"
+    log "$name: forget $prune"
+    restic forget --host {{ inventory_hostname }} {{ restic_keep }} $prune || failed=1
+  done
+
+  rm -rf "$STAGE"
+  exit $failed
+  ;;
+
+*)
+  [ -n "${1:-}" ] && [ -n "${REPOS[$1]:-}" ] || { sed -n '4,10p' "$0"; exit 2; }
+  export RESTIC_REPOSITORY="${REPOS[$1]}"; shift
+  exec restic "$@"
+  ;;
+esac

ansible/roles/restic_backup/templates/restic-backup.service.j2

@@ -0,0 +1,12 @@
+[Unit]
+Description=restic backup to Hetzner Object Storage ({{ restic_repositories.keys() | join(', ') }})
+Wants=network-online.target
+After=network-online.target docker.service
+OnFailure=restic-alert@%n.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/restic-backup run
+Nice=10
+IOSchedulingClass=idle
+TimeoutStartSec=6h

ansible/roles/restic_backup/templates/restic-backup.timer.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=Nightly restic backup
+
+[Timer]
+OnCalendar={{ restic_schedule }}
+RandomizedDelaySec=10m
+Persistent=true
+
+[Install]
+WantedBy=timers.target

ansible/site.yaml

@@ -36,6 +36,12 @@
   roles:
     - { role: forgejo, tags: [forgejo] }
 
+- name: Backups
+  hosts: asxpio
+  become: true
+  roles:
+    - { role: restic_backup, tags: [restic] }
+
 # Last: when the pipeline changes the runner it recreates the container
 # that is running the pipeline. Everything else is done by then; that job
 # ends as failed and the next run confirms the result.