From 59b5b4468aa64bf41c201e4bfcde7de9d95b6e39 Mon Sep 17 00:00:00 2001 From: Sergei Poljanski Date: Thu, 12 Nov 2026 20:53:44 +0400 Subject: [PATCH] ansible: nightly restic backups to both object storage regions Hetzner's server backups cost 20% of each server's price and only restore whole disks. Back up what matters with restic instead, into the backup buckets we already pay for: one repository per host in hel1 (asxpio-backups-main) and one in fsn1 (asxpio-backups-repl), written independently so either region alone is a full copy. Postgres goes in as a dump, Forgejo's and vhpbot's SQLite databases through the online backup API, Garage with a fresh metadata snapshot, ejabberd through its own backup command. Keeps 7 daily, 4 weekly and 6 monthly snapshots, prunes on Sundays, and mails the journal when a run fails. restic is the pinned upstream release, so its version is tracked in the repo like everything else. The vault gains the restic password and the Xray config for the role that follows. Assisted-by: Claude:opus-5-5 --- ansible/inventory/group_vars/all/vault.yaml | 258 +++++++++++------- ansible/inventory/host_vars/asxpio-ops.yaml | 19 ++ .../inventory/host_vars/asxpio-services.yaml | 18 ++ .../roles/restic_backup/defaults/main.yaml | 36 +++ .../roles/restic_backup/files/restic-alert | 20 ++ .../restic_backup/files/restic-alert@.service | 6 + .../roles/restic_backup/handlers/main.yaml | 4 + ansible/roles/restic_backup/tasks/main.yaml | 88 ++++++ ansible/roles/restic_backup/templates/env.j2 | 8 + .../restic_backup/templates/restic-backup.j2 | 70 +++++ .../templates/restic-backup.service.j2 | 12 + .../templates/restic-backup.timer.j2 | 10 + ansible/site.yaml | 6 + 13 files changed, 450 insertions(+), 105 deletions(-) create mode 100644 ansible/roles/restic_backup/defaults/main.yaml create mode 100644 ansible/roles/restic_backup/files/restic-alert create mode 100644 ansible/roles/restic_backup/files/restic-alert@.service create mode 100644 ansible/roles/restic_backup/handlers/main.yaml create mode 100644 ansible/roles/restic_backup/tasks/main.yaml create mode 100644 ansible/roles/restic_backup/templates/env.j2 create mode 100644 ansible/roles/restic_backup/templates/restic-backup.j2 create mode 100644 ansible/roles/restic_backup/templates/restic-backup.service.j2 create mode 100644 ansible/roles/restic_backup/templates/restic-backup.timer.j2 diff --git a/ansible/inventory/group_vars/all/vault.yaml b/ansible/inventory/group_vars/all/vault.yaml index 7f73a5b..e408ee2 100644 --- a/ansible/inventory/group_vars/all/vault.yaml +++ b/ansible/inventory/group_vars/all/vault.yaml @@ -1,106 +1,154 @@ $ANSIBLE_VAULT;1.1;AES256 -37373533663231633738353864666365663334666134346462306537373463616438663334613764 -6264643666356130396536323235333837316431373536370a623833383939373038366263393161 -62623563386130666637353164363331373061643563313861643239313363323234303738343738 -3864363533393962650a633633613138313066336238336363306161306430626165343039356664 -64643333633938306138636334656631316232626539336466313033373430623263376635626238 -33343839656639366363313535663561666535376461616366376232303034343064626334626432 -36663362386561616537343835373839363632363730386666653066343731643330623266353434 -39306664363561316138363862356663366365643738336536353137396463623630343430346137 -63336164393939666266306536363435663661663939333238373639346431636635323834356437 -31316632333731343362303331353338333830356635643431316165633963356264376134623665 -37626263336339386234376534366463646537623631303462366466633034343932346132363532 -65313237303966333864373664303931653833323036316434666338353430303333386338356262 -62633533613061653262323630366230303561623661303233316131373066343666303635323637 -63313739623230336461613530623732636262393239306663643835396365636166616536316336 -39376163633937353564343766326462663362396239383835363030393539393936383535353561 -64643231346465393261626266666532636265356531356463393839636265613431383335663263 -32353864333335333632653461366338386664373939623234306466323933316161633364353437 -31626164386430656432363861316464313336613466383562663838393738633337303139386264 -61363933653237653237353836313733666266666534333331376633643938323264623664663733 -38616565343830616638623032646533653433663935363563336665623638313637643337646439 -63666661366463313131363433643461626361303130653761333538303739653265646263626263 -37373137373038643365326566383237393236613831393833326630386237316133333935336234 -32353732633334393830306435313435363538626566386135326338363630666561623633323865 -62313732363562643766386462646531383238363962313437373735313262633162643666303136 -30393332663865313064653233653761333364306134646163373365376532393838666262366535 -38633135363932636533333139386136363964316538316435343633666637306139313033666531 -39396138346461333238623362343162663366333738616663353963653061663531343230326265 -62353466393564666462633462303939346532323931306533323132643862653265346137353738 -63366238323130656430313831326330623961383237346436633165366634356466636162646261 -30303962386532353739643464653139386336393737316266653764373738373530323863326233 -36383135366336333831383438323261636665666331663338613266666338623630363533366338 -62626231306135643830336235353338396336646238396633363561326163623132653638306131 -39356634313130373862356263393364623337363136626361396536616631326233343730643864 -63633633343662343833333064356432373235653864303863656163643830333339656265646639 -65623636333630643236316434326264356435393837626330383962336161633262356638393266 -66323337346434333563323032393563663831653437336634613665306331366662623530393263 -34313033303966633235323339356461356630636238313031633834383434343035353438383035 -32386462323232623861626530373663333564333932616563613432383134393661303264303130 -30353431353062373533303237653966373762346264613734643730663932643633353631666565 -33633337633561343733623434653862383364646434663333366334323739653933313537363366 -65396532346439653334653739376562343136346266333739396630373166613763313563396133 -30303530326465653339343035623333646238393034323862623765633139396364346339326233 -36656664303536623834326662666537343537643336366138383935343065313538613363373966 -35393961356531623262363163343034656130383235373937643639613331356333323964633966 -65343632303237653939393438636137316332313931353934313034326366623832353363636564 -66363437616663356431613063613535666336383730333337623439653130653439663830633534 -39356266346231323535646265646439326137346361656632323564313435626165643231653735 -62636530383961343662393232666664363034656534663762393262356163336337373862316331 -62346535636464633363663330323335386230633465613965613630613135366131313138653061 -36613930653430643236323162643864353863323264323731333037306433376236306138303032 -65333963333364326665616333353031323761653764316362336337386230633736363766333662 -64316662343265623064643031326534346639353566376661343462613537373862303233316538 -64323364366430326264366232666263396235363736366437346438656265386464323264653733 -63363263636233353761356331613330353535346336366437613837343061663362663031333964 -34396366313363383863303035633934376432393334663364313866396166623563646530663632 -64613336653635636564373632646337386438333431326264373766306564326430326237363964 -65386631393135313566656435353039363638393535613063393730316634313837623630323161 -36356331323165613537386164343831386633663936313537306639633262633239353830313530 -31363066316332373035656134333937373263633561616431323334336335656131383436626464 -64356634626636356331383962643735303335363863343561373736363134646536393431346432 -62373434323665343533636237363461303335663632326332633535303139393433623032613465 -66383137613531376333323431356566653462616330656532633361623338343637613239303433 -62626537373165666562646561373731383436353436323434653562396334313639616661643935 -30616431313936626637353966366531316463616366306335656664613366303665346263633830 -31303939623631636463366132613866343032633365326433356463643832353230653339646161 -34636637373464386162623961373933363863346637313535643134613265383166393634656131 -64343930643162613362386465306465333235663461303566366266303664316565396563333433 -61356130333833653330666633346663373634396634383030346138313938636134303666623962 -62356636633062646639323963653565386464373736626161633565303335336262383230363732 -39393539333262326234333537356461663033303933383237346235373734656536303862333236 -65643563343932333632386635346438323339343463323338383865343362333430346338633932 -32386139343866633139313533663832356633633663616261386364373065373365386334383932 -63663334616532316133353565643936643035346464643936643862333466363166333066326338 -30326462333463393834303836653437353033313736643365313739313334383030636535656434 -32373231366462643233363837373133366139366130393537363361346133653839396432343936 -34303030373638303934343463326365663466633963323033323066383734336331353864313564 -64393638353735646233326638346464336637373465663562336661363834343138376635323363 -65356537356137336563383866356665383132366136633035646363393933643165656236313863 -30386363393466636435643666663533623332656232393537656161613837663064353838653835 -66666164386566663536653833613062333432363966303066323563363664333464633038353561 -32626634643264373132323638643766386362613639303262633462663737663337393631303436 -62326438396235666338313033383835306566366365396331653633323335333865366565343036 -62313835333964366537323565653662323563383663323033623939346533656665303235313264 -32623163383530376233353738633062346530376161353664333166323538666465363237616166 -34633561636234623263663535333939306239303334363939616636343334383838643939636664 -63623331306332386634646533396232323063336166613339636533353337306531636135313465 -39636164323537643230353234303666313161633366633831356530633635636561373461643136 -66323734386239656635633365373734643038636566626666646434663061333530633837633166 -38663633393764346565316639323638623039326332646432666232656463626335313737623862 -66386439616635633061356461343736656138633064653733643737636361626366313733356665 -34346565613330663938333563623436323934353934393235356234613766633235316137376632 -32386333373235393837393035653538303433373733386562623037363837396431393531303839 -31633162386637333963633335383662623661336133653866653336376366363665623165633032 -65636261616664326661333930343862663766393035613164313837333561326330333930643166 -38393161316364306139376234643363346564613463613536613534373936653532353861633139 -39663130316130366333303834386162343336346664316439373839333465356564303062633434 -62626337616366303730663661383863653834623537333933616239386264326634656235363166 -63323061343639666135306437356665313466326339356230376234363964653631643561633236 -31386339343033353062383231666135303338356335366335663763383237626138633961393239 -66363539333161313138626332303662366336323266643062663762323561656634636539336265 -34653736306231356135626435653736366133386538643338346463653231653333306633666532 -61323266303764653734643262363564656233653135623234663232353762653766313565613736 -31613432363333366233323165616561646366643634376639343966373635343335343661636462 -32613563613139363932636330663764353164316231333461653965373635363337633662653962 -35346463643363376633356234626464666332646331643132643138346665393937 +32356366663237323730376532393462356237346236616131303336623262653236356331333330 +3035633139313236643830343031323136363663383338350a613731626164323234656637653630 +61613839303334303866613762346363373731333962323066666132313664616438373365613635 +3139633033346437390a643430646134376364393737343234386661353938623531353937666430 +65366364613362393262313863346432306265353264353464313335646365326233376631373732 +63666661363462663633633461323635633065613633636664383233353661643638316662386633 +61626635386463323230366565386163393063383436636666643037353337343339373035366539 +31353561386565383232306263333231326261633132303036666562326635313365346633636634 +32353739633631666134373963383865363935326130336336333739393130353831646632623562 +61373038376462363864333061626564623239396135626537366439373063313039313966363439 +39363837356434356166353839363035373233656665663838346563623433346333643038323130 +35333239313863303066366265373365636535313339616238343435613737356635643833633066 +39613835306532383930303130633535666238323864386466336131313637336361313137306161 +61653063663131666539363463326332383763643064653133343633633066366362373235663736 +65376239316364666161323661613661333265366432366137386538646333653166646538626364 +30636263336235336363646136663266376538646631323665636561313537393834336565313033 +65663339626638303365333438393261653432373438373065343862373339623361383530343232 +64333964643837333666663434356534653561313864383137303761346364633461653635656566 +32613434373365616132353061333864333063386335653638623165333765653230313939353037 +38633966323032623937376565383134383034376539383632393064666464313439633330666335 +34313038333366353433646236666265366235373439626162306435663364643333316562346439 +32653466613966303933633538366466636461373930353364323030646436316436383562646664 +62626230363830306366613530633363653265623531383936396539356132393634323434356437 +66636661626335323737643538343834303238346631666334393934343134366331346532396137 +39396662396437656138626230343438336637663163303735653335663134343862373862386465 +63643362643462663961663165306533623931313133636365616665333537643365623833303439 +30656338323062366263636666333636356565653338323366346431396434633261623833376635 +62346563373163663163643036383730663939613032663161396432653039373561663264646135 +36303836656563333431633565383130616335356539333934313065333638363739393930393232 +65366537623836356361393865396534376238663264643834333230636636666139643734336161 +63333164303132373136623834666566663864336331666533646130393465613039613166306135 +63373334346134396566656233393666333063313435626561356464363064643966373561346465 +30346334383766623539386338363363343938343166633834326261396637333963656437663137 +65633233383138666538663933303337623130613166373632623334386236313765633037613165 +65653565623264623965303133653938383261636163373238353632313365613264643834656330 +34383965666337663334333362623266356666643565626432383861366131336564353732356634 +62316164663463326539316264323433346430363431383131393832383530333634643065376536 +32616132626332623132333061353961396331366666326631303165303261326465303062323863 +34656164613132616138383537336466353461363233393161366566356664336331313661383835 +38393938633537313731626132376636343234336431393939643438363634326434333236333335 +36356538653465616663633337626439366132383434326539663434333133356364653133336639 +33313331373761643662313431386364303262383566333462323532343135376666333363333432 +61656234643666343531646665336534343533346530326265643335383962376662326631653638 +32626439303562623364353263316535646134363736323133393832653536353262306334393439 +32306531623134656438613738643562663339336438353537343837393164386338363263323762 +63333162316365323964313663383863363836643636326237626432623738323063303639373639 +65363330353736333831333431393332616264353263316338383165303237343037396362346265 +62663330643964663833346561363661613137646563396164383236633135343136653338353664 +35616164646332333338626336376138303133323335353031333566613732386335386365626635 +62353138343165346437666233633736646332313132396462633235323563636338333662636466 +37343364343533623034653165633263343238666265636365636162646262383834613238366635 +62656233653162346530393236343437623137313766306466646136643962343062643732306164 +38633432363365613864376436393566363239343761623838623232613033363438656136633231 +39303366623764616137646136353336343765353266363161343638303532356261666166616138 +64373865303035643866666335356639656134353339383666373332363631393030346433316466 +39663535393238613537343764316634633632616465343539643938313134653437386132336166 +32636365353161313030393830653565623933313165323661376635643638646237313661336566 +31323533353536306561663735383639353539663835366632333232343038646432666438326263 +35386162663264396562613338386336623536306530366266346336643763656532653733366534 +64393436313962356233323164383639363435656639303563393336623735613363373933313139 +37353930396531633531336333656339336264353864396461393162316365343261653939313039 +61653164646466613562386566663031356363383836633661373566386235626339646139393234 +61323237383131386531616463663465613731306636356230306530623261613838633562333031 +61326131393631633161386363386635633465613430616336356537616664383763333566363033 +35363634643537646337363639333039343761363032633934633339633765643330613035633939 +38353133646461653038383837656431306239306366323866623539383561316432626135386234 +63353533336330643263346338353538383663656161616162363538323061613237396332303866 +35326466663762616138373839383865316131346438353038626134313861613139336162616138 +33393665396439323738613030366439323136653164616163643433343438373131393666633638 +36333236323434376533626435353264663930663130343037616462306663616139363565313732 +32666166643830303536633439353130303235323166626634313139613565323235666131303432 +33316531623736333832356563383234346338663634366436306533366464366663646137396566 +31393034366439646161643361663436383437623839323366616537353436366366363036373565 +37396439633238313266383635623665396636333431313734303435323134356163656462323964 +39313335356133623234393536346539333231373035396233343832633566643337656237623465 +35623764343934646261333266613562363630396532653764303531383666363766383765363734 +37663537613633643830326237363335623566376639633065363330313462633566653832643866 +35633434303665656462653438313838636638333734376539383931396535333837393330333739 +34313434653366663737616331626136626634646333303637363563336466303938363630333334 +30656264633065333137643865633030643230636363396565383530653734653134303032376231 +30643834636361313131653266303633626565666234336638386538633739376238343831343035 +35373265343966666462633261323538633531653238346466616234313537323938316239383162 +35326438313065313534313534626535313738323035386163663666653339613166373837346532 +65623235353136323961623631636437656234633337653036353461313939373464383431343163 +39333836336462636562396161303636666238663432343838383833306130633639663262616361 +30323331386363373262356666303036343836343636353865613965643365396238313564333966 +36363031386637313230303536303232613331666334393535363365653733353965396436373561 +63636137313836346164383239363862353830333038363562396133343536633335633235333133 +32636263633635366664363235666330633364303531376166666530646336346664323338636262 +39643534353535323534366164633435343963333331353233633432623738313038326132363962 +63316563376163386139656566656661396664666231373433613337326238303362383563623661 +63353762336336303362383132643766663138313739366634666531616363616161613836666665 +31396265353033663166366137393030366265333232663161356636343863333633646166313466 +61636238613366623835383633346538306130626132346239346166653736343138613633373765 +39623437623531313138313731313934623837313039363233653930646336393934373361303834 +66373464323066303735663862383266346131323365313932356132633037366337373136313434 +39373832346264363633306535623733363666393032353031373336353239646461306162653534 +63323132626362316538323264316362656233346130326165333334333834643333636234626530 +35633531663034636637373762333632336262373066356530613631343664633764643765613131 +66353036356664646264336131323539633038373438623261313335663763386334336662376634 +35376661316333363539633239653337663064613033363931653362323633386664366138363137 +30336536343430333936656536663331656631613733396632343764363163313862373936323663 +36306432663539336531663738363365376431616262346664613134366664373536386433333566 +31346133643639613934383964656162623037373462633031353561623238636230633731323536 +31326562343339376664653065656635353662663634386263363330356566336661313536666265 +66666165356534306163326137336331306430336536663965323964656233396638636563356365 +31366161363366656463373063636433343039303037646533626633613362393639323736623437 +30386665326534623266633463643535356633303639346632666633363464306230653837333937 +32636538353431633963366132366331323838393639613535613237646466666532386535323966 +38323363353764313332336137653364343232323164353733646631343565316133356436343936 +35636132663964633135363335383062663464346330666164396330616435663663363664643064 +38306162616232633239313565383836356566376238303164393539666265313066346632303165 +62393866346363653831623063346631396631653863303038373239643964666238333461303734 +33343136333535616332633038646663623232663730376438386538633936323033356534356532 +30313638326633396261343937333464386135646231653262646162303739613232326139303034 +39326633383937313536636466373561393866303664653264353033363039613664656637656539 +32616230663138323964663236363763643833373161313066363434636334636337323035643131 +63636131663537313438316632376134313666393437303031356662626531653639303363346335 +36393862366433343365643738666132666130303331303837306432623731383863613230373130 +65623030366661653439303561356533336465656636356164646636373765336239353461636566 +35616335643430663666653363666431356436343439643464356635313764663335613335396161 +62393661353733336535633436626366653031323663383133383930383034313236353362376637 +35323438633761383366383066363663373563656465316134373162396130616463333634376431 +32323465356535653938303732663464653966613834313463616161343637306236393266656565 +64666465313665633064343639656532633331663362366434393566643630333732346465626237 +38373433623037396364663165623263326262323962616636613865393536626661356133613764 +66633036623335646166356537313337353634326239646262303964633836636463343735313966 +34386335393564613961623231336638313835613935316335306438343565333330303866646636 +31663731663762643136323062643433636666626330376364396230663765613466373336303730 +65326434393465336532353236366139633935666130653735346265643463366563306266666537 +34636662386238343032616263363736643435313839393038643934313263613635366537623666 +33623034386166663831616563343733653866326263396261636635363337663239366261623538 +31363132356166363664653466626238636565633265346631626632356337303631613432346435 +39666137653636393135306135313465313035656538336434393233663361326139643237626439 +39613461333766333939636461323430623137383434343563333734336236346336626337313736 +31396535633136623333336465366166376338376235623537366636636331653737346165656364 +62663831323530626133323138393231306534323166643430656237336537306231623131383839 +65306661326533326539646266363461373461646464366264393364333232653535333934623738 +66343266333530613032316633323662356339366635373166623833646130643337613436363537 +39353737303862666234323230303137623434633636303263653762393937613234383862313561 +39613138356237373639663565653664643265373133356436343332363961616438643937393735 +31653461353030626664393137343431663264636263326535646665373130313961316639636631 +33626533336362356631366133636666343236396263626133626132343762633163633339313663 +65643930383164353931646538653065616362636335333236353764643864613338623465633066 +31333135633963383361623533316636346434316137633235343134663266373234383664323539 +31306233303234336130346665356133326431306430366364636662663630663833303664356632 +36633263653435333261313065653634653736393837306633393936386561613838343736326331 +65613233343866313835313733613630363736613064366138326364636566343961663338376231 +30636538303030303038353637663732336136656263303737623664376165393861643831616464 +37643433316634336238333432643039643334366632326234646339356333663739633934323632 +36626230653437303762343039643235646432663831363338643466396439376537353964663464 +31353433363037363135316535643336653430383333633465376631653162373630323636636365 +35376531633239393636396363626432306565656432653938396535663739386239 diff --git a/ansible/inventory/host_vars/asxpio-ops.yaml b/ansible/inventory/host_vars/asxpio-ops.yaml index 18a613a..7aa3f06 100644 --- a/ansible/inventory/host_vars/asxpio-ops.yaml +++ b/ansible/inventory/host_vars/asxpio-ops.yaml @@ -10,3 +10,22 @@ ssh_git_user: git # s3fs is off here: its ensure_diskfree reserve never fit the 38 GB disk s3fs_enabled: false s3fs_ensure_diskfree: 20000 + +# restic: Postgres as a dump, Garage with a fresh metadata snapshot, +# Forgejo's SQLite through the online backup API +restic_paths: + - /etc + - /opt + - /root/backups/rclone-hetzner.sh + - /var/lib/docker/volumes/storage_garage_meta + - /var/lib/docker/volumes/storage_garage_data +restic_excludes: + - /opt/backups + - /opt/containerd +restic_pre_hooks: + - docker exec storage-postgres pg_dumpall -U postgres > "$STAGE/pg_dumpall.sql" + - docker exec storage-postgres pg_dump -U postgres -Fc asxpio > "$STAGE/asxpio.dump" + - docker exec storage-garage /garage meta snapshot --all + - sqlite3 /opt/forgejo/data/gitea/gitea.db ".backup '$STAGE/forgejo-gitea.db'" + - sqlite3 /opt/vhpbot-prod/bot.db ".backup '$STAGE/vhpbot-prod.db'" +restic_schedule: "*-*-* 03:30:00" diff --git a/ansible/inventory/host_vars/asxpio-services.yaml b/ansible/inventory/host_vars/asxpio-services.yaml index 02be751..50000e7 100644 --- a/ansible/inventory/host_vars/asxpio-services.yaml +++ b/ansible/inventory/host_vars/asxpio-services.yaml @@ -6,3 +6,21 @@ wireguard_ip: "{{ asxpio_services_wireguard_ip }}" ssh_allow_public_git: false s3fs_ensure_diskfree: 81920 # 80 GB free = max 80 GB cache + +# restic: ejabberd's Mnesia through its own backup command, its SQLite +# through the online backup API. The Paper world is copied live (RCON is +# off, so there is no save-off); its autosave keeps that close enough. +restic_paths: + - /etc + - /opt + - /var/lib/docker/volumes/xmpp_database + - /var/lib/docker/volumes/xmpp_tackygram +restic_excludes: + - /opt/containerd + - /opt/minecraft/cache + - /opt/minecraft/libraries + - /opt/minecraft/versions +restic_pre_hooks: + - docker exec xmpp-ejabberd-1 ejabberdctl backup /opt/ejabberd/database/nightly.backup + - sqlite3 /var/lib/docker/volumes/xmpp_database/_data/ejabberd.sqlite ".backup '$STAGE/ejabberd.sqlite'" +restic_schedule: "*-*-* 04:15:00" diff --git a/ansible/roles/restic_backup/defaults/main.yaml b/ansible/roles/restic_backup/defaults/main.yaml new file mode 100644 index 0000000..9fb0ef1 --- /dev/null +++ b/ansible/roles/restic_backup/defaults/main.yaml @@ -0,0 +1,36 @@ +--- +# Nightly restic backup of the host to Hetzner Object Storage, one +# repository per region, written independently: losing one region or one +# repository leaves the other complete. + +# renovate: datasource=github-releases depName=restic/restic +restic_version: "0.19.1" + +restic_password: "{{ vault_restic_password }}" +restic_s3_access_key: "{{ s3_access_key }}" +restic_s3_secret_key: "{{ s3_secret_key }}" + +restic_repositories: + hel1: "s3:https://hel1.your-objectstorage.com/{{ s3_bucket_backups_main }}/restic/{{ inventory_hostname }}" + fsn1: "s3:https://fsn1.your-objectstorage.com/{{ s3_bucket_backups_repl }}/restic/{{ inventory_hostname }}" + +# Dumps and snapshots taken by the hooks land here and are backed up too +restic_stage_dir: /var/backups/restic-stage + +restic_paths: [/etc, /opt] +restic_excludes: [] +# Shell lines run in order before the backup, with $STAGE set. A failing +# hook marks the run as failed but the backup still runs. +restic_pre_hooks: [] + +restic_keep: "--keep-daily 7 --keep-weekly 4 --keep-monthly 6" +restic_schedule: "*-*-* 03:30:00" +# Prune (rewrites pack files, so costs S3 requests) once a week only +restic_prune_weekday: Sun + +# Failure mail, through Forgejo's SMTP account +restic_alert_smtp_host: smtp.fastmail.com +restic_alert_smtp_port: 587 +restic_alert_smtp_user: "{{ vault_forgejo_smtp_user }}" +restic_alert_smtp_password: "{{ vault_forgejo_smtp_password }}" +restic_alert_recipient: "{{ vault_crawler_recipient }}" diff --git a/ansible/roles/restic_backup/files/restic-alert b/ansible/roles/restic_backup/files/restic-alert new file mode 100644 index 0000000..096bfd3 --- /dev/null +++ b/ansible/roles/restic_backup/files/restic-alert @@ -0,0 +1,20 @@ +#!/usr/bin/python3 +# Mail the tail of a failed unit's journal. Usage: restic-alert +import os, smtplib, socket, subprocess, sys +from email.message import EmailMessage + +unit = sys.argv[1] +env = dict(l.rstrip("\n").split("=", 1) for l in open("/etc/restic/env") if "=" in l) +log = subprocess.run(["journalctl", "-u", unit, "-n", "60", "--no-pager", "-o", "cat"], + capture_output=True, text=True).stdout + +msg = EmailMessage() +msg["Subject"] = f"[{socket.gethostname()}] {unit} failed" +msg["From"] = env["ALERT_SMTP_USER"] +msg["To"] = env["ALERT_RECIPIENT"] +msg.set_content(log) + +with smtplib.SMTP(env["ALERT_SMTP_HOST"], int(env["ALERT_SMTP_PORT"]), timeout=30) as s: + s.starttls() + s.login(env["ALERT_SMTP_USER"], env["ALERT_SMTP_PASSWORD"]) + s.send_message(msg) diff --git a/ansible/roles/restic_backup/files/restic-alert@.service b/ansible/roles/restic_backup/files/restic-alert@.service new file mode 100644 index 0000000..498970e --- /dev/null +++ b/ansible/roles/restic_backup/files/restic-alert@.service @@ -0,0 +1,6 @@ +[Unit] +Description=Mail the failure of %i + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/restic-alert %i diff --git a/ansible/roles/restic_backup/handlers/main.yaml b/ansible/roles/restic_backup/handlers/main.yaml new file mode 100644 index 0000000..c28484f --- /dev/null +++ b/ansible/roles/restic_backup/handlers/main.yaml @@ -0,0 +1,4 @@ +--- +- name: Reload systemd + ansible.builtin.systemd: + daemon_reload: true diff --git a/ansible/roles/restic_backup/tasks/main.yaml b/ansible/roles/restic_backup/tasks/main.yaml new file mode 100644 index 0000000..dc2048e --- /dev/null +++ b/ansible/roles/restic_backup/tasks/main.yaml @@ -0,0 +1,88 @@ +--- +- name: Install sqlite3 for consistent copies of live databases + ansible.builtin.apt: + name: [sqlite3, bzip2] + state: present + +- name: Check the installed restic version + ansible.builtin.command: /usr/local/bin/restic version + register: restic_installed + changed_when: false + failed_when: false + check_mode: false + +- name: Install restic {{ restic_version }} + when: ("restic " ~ restic_version ~ " ") not in (restic_installed.stdout | default('')) + block: + - name: Download the release + ansible.builtin.get_url: + url: "https://github.com/restic/restic/releases/download/v{{ restic_version }}/restic_{{ restic_version }}_linux_amd64.bz2" + dest: "/tmp/restic_{{ restic_version }}.bz2" + checksum: "sha256:https://github.com/restic/restic/releases/download/v{{ restic_version }}/SHA256SUMS" + mode: "0600" + + - name: Unpack it + ansible.builtin.shell: > + bunzip2 -c /tmp/restic_{{ restic_version }}.bz2 > /usr/local/bin/restic.new && + chmod 755 /usr/local/bin/restic.new && + mv /usr/local/bin/restic.new /usr/local/bin/restic && + rm /tmp/restic_{{ restic_version }}.bz2 + changed_when: true + +- name: Create config and cache directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + mode: "0700" + loop: [/etc/restic, /var/cache/restic] + +- name: Write credentials + ansible.builtin.template: + src: env.j2 + dest: /etc/restic/env + mode: "0600" + diff: false + +- name: Install the scripts + ansible.builtin.template: + src: restic-backup.j2 + dest: /usr/local/sbin/restic-backup + mode: "0700" + +- name: Install the alert script + ansible.builtin.copy: + src: restic-alert + dest: /usr/local/sbin/restic-alert + mode: "0700" + +- name: Install the systemd units + ansible.builtin.template: + src: "{{ item }}.j2" + dest: "/etc/systemd/system/{{ item }}" + mode: "0644" + loop: [restic-backup.service, restic-backup.timer] + notify: Reload systemd + +- name: Install the alert unit + ansible.builtin.copy: + src: restic-alert@.service + dest: /etc/systemd/system/restic-alert@.service + mode: "0644" + notify: Reload systemd + +- name: Flush handlers + ansible.builtin.meta: flush_handlers + +- name: Create the repositories + ansible.builtin.command: /usr/local/sbin/restic-backup init + register: restic_init + changed_when: "'created' in restic_init.stdout" + when: not ansible_check_mode + +- name: Enable the nightly timer + ansible.builtin.systemd: + name: restic-backup.timer + enabled: true + state: started + # On a first dry run the unit is not installed yet + ignore_errors: "{{ ansible_check_mode }}" diff --git a/ansible/roles/restic_backup/templates/env.j2 b/ansible/roles/restic_backup/templates/env.j2 new file mode 100644 index 0000000..9b36af8 --- /dev/null +++ b/ansible/roles/restic_backup/templates/env.j2 @@ -0,0 +1,8 @@ +RESTIC_PASSWORD={{ restic_password }} +AWS_ACCESS_KEY_ID={{ restic_s3_access_key }} +AWS_SECRET_ACCESS_KEY={{ restic_s3_secret_key }} +ALERT_SMTP_HOST={{ restic_alert_smtp_host }} +ALERT_SMTP_PORT={{ restic_alert_smtp_port }} +ALERT_SMTP_USER={{ restic_alert_smtp_user }} +ALERT_SMTP_PASSWORD={{ restic_alert_smtp_password }} +ALERT_RECIPIENT={{ restic_alert_recipient }} diff --git a/ansible/roles/restic_backup/templates/restic-backup.j2 b/ansible/roles/restic_backup/templates/restic-backup.j2 new file mode 100644 index 0000000..e0d10aa --- /dev/null +++ b/ansible/roles/restic_backup/templates/restic-backup.j2 @@ -0,0 +1,70 @@ +#!/bin/bash +# Managed by Ansible (role restic_backup). +# +# restic-backup run hooks, then backup + forget to every repository +# restic-backup init create the repositories that do not exist yet +# restic-backup plain restic against one repository, e.g. +# restic-backup hel1 snapshots +# restic-backup fsn1 restore latest --target /tmp/r +# +# Repositories: {{ restic_repositories.keys() | join(', ') }} +set -uo pipefail + +set -a; . /etc/restic/env; set +a +export RESTIC_CACHE_DIR=/var/cache/restic +STAGE={{ restic_stage_dir }} + +declare -A REPOS=( +{% for name, url in restic_repositories.items() %} + [{{ name }}]="{{ url }}" +{% endfor %} +) + +log() { echo "restic-backup: $*"; } + +case "${1:-}" in +init) + for name in "${!REPOS[@]}"; do + if RESTIC_REPOSITORY="${REPOS[$name]}" restic cat config >/dev/null 2>&1; then + log "$name: exists" + else + RESTIC_REPOSITORY="${REPOS[$name]}" restic init || exit 1 + log "$name: created" + fi + done + ;; + +run) + failed=0 + rm -rf "$STAGE"; install -d -m 700 "$STAGE" + export STAGE +{% for hook in restic_pre_hooks %} + log "hook {{ loop.index }}/{{ restic_pre_hooks | length }}" + ( set -e; {{ hook }} ) || { log "hook failed"; failed=1; } +{% endfor %} + + for name in "${!REPOS[@]}"; do + export RESTIC_REPOSITORY="${REPOS[$name]}" + log "$name: backup" + restic backup --host {{ inventory_hostname }} --one-file-system --exclude-caches \ +{% for e in restic_excludes %} + --exclude '{{ e }}' \ +{% endfor %} + {{ (restic_paths + [restic_stage_dir]) | join(' ') }} || { failed=1; continue; } + + prune="" + [ "$(date +%a)" = "{{ restic_prune_weekday }}" ] && prune="--prune" + log "$name: forget $prune" + restic forget --host {{ inventory_hostname }} {{ restic_keep }} $prune || failed=1 + done + + rm -rf "$STAGE" + exit $failed + ;; + +*) + [ -n "${1:-}" ] && [ -n "${REPOS[$1]:-}" ] || { sed -n '4,10p' "$0"; exit 2; } + export RESTIC_REPOSITORY="${REPOS[$1]}"; shift + exec restic "$@" + ;; +esac diff --git a/ansible/roles/restic_backup/templates/restic-backup.service.j2 b/ansible/roles/restic_backup/templates/restic-backup.service.j2 new file mode 100644 index 0000000..b7d2384 --- /dev/null +++ b/ansible/roles/restic_backup/templates/restic-backup.service.j2 @@ -0,0 +1,12 @@ +[Unit] +Description=restic backup to Hetzner Object Storage ({{ restic_repositories.keys() | join(', ') }}) +Wants=network-online.target +After=network-online.target docker.service +OnFailure=restic-alert@%n.service + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/restic-backup run +Nice=10 +IOSchedulingClass=idle +TimeoutStartSec=6h diff --git a/ansible/roles/restic_backup/templates/restic-backup.timer.j2 b/ansible/roles/restic_backup/templates/restic-backup.timer.j2 new file mode 100644 index 0000000..42fae04 --- /dev/null +++ b/ansible/roles/restic_backup/templates/restic-backup.timer.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Nightly restic backup + +[Timer] +OnCalendar={{ restic_schedule }} +RandomizedDelaySec=10m +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/site.yaml b/ansible/site.yaml index f1a81ca..f5f7ffa 100644 --- a/ansible/site.yaml +++ b/ansible/site.yaml @@ -36,6 +36,12 @@ roles: - { role: forgejo, tags: [forgejo] } +- name: Backups + hosts: asxpio + become: true + roles: + - { role: restic_backup, tags: [restic] } + # Last: when the pipeline changes the runner it recreates the container # that is running the pipeline. Everything else is done by then; that job # ends as failed and the next run confirms the result.