diff --git a/ansible/inventory/group_vars/all/vault.yaml b/ansible/inventory/group_vars/all/vault.yaml index 7f73a5b..e408ee2 100644 --- a/ansible/inventory/group_vars/all/vault.yaml +++ b/ansible/inventory/group_vars/all/vault.yaml @@ -1,106 +1,154 @@ $ANSIBLE_VAULT;1.1;AES256 -37373533663231633738353864666365663334666134346462306537373463616438663334613764 -6264643666356130396536323235333837316431373536370a623833383939373038366263393161 -62623563386130666637353164363331373061643563313861643239313363323234303738343738 -3864363533393962650a633633613138313066336238336363306161306430626165343039356664 -64643333633938306138636334656631316232626539336466313033373430623263376635626238 -33343839656639366363313535663561666535376461616366376232303034343064626334626432 -36663362386561616537343835373839363632363730386666653066343731643330623266353434 -39306664363561316138363862356663366365643738336536353137396463623630343430346137 -63336164393939666266306536363435663661663939333238373639346431636635323834356437 -31316632333731343362303331353338333830356635643431316165633963356264376134623665 -37626263336339386234376534366463646537623631303462366466633034343932346132363532 -65313237303966333864373664303931653833323036316434666338353430303333386338356262 -62633533613061653262323630366230303561623661303233316131373066343666303635323637 -63313739623230336461613530623732636262393239306663643835396365636166616536316336 -39376163633937353564343766326462663362396239383835363030393539393936383535353561 -64643231346465393261626266666532636265356531356463393839636265613431383335663263 -32353864333335333632653461366338386664373939623234306466323933316161633364353437 -31626164386430656432363861316464313336613466383562663838393738633337303139386264 -61363933653237653237353836313733666266666534333331376633643938323264623664663733 -38616565343830616638623032646533653433663935363563336665623638313637643337646439 -63666661366463313131363433643461626361303130653761333538303739653265646263626263 -37373137373038643365326566383237393236613831393833326630386237316133333935336234 -32353732633334393830306435313435363538626566386135326338363630666561623633323865 -62313732363562643766386462646531383238363962313437373735313262633162643666303136 -30393332663865313064653233653761333364306134646163373365376532393838666262366535 -38633135363932636533333139386136363964316538316435343633666637306139313033666531 -39396138346461333238623362343162663366333738616663353963653061663531343230326265 -62353466393564666462633462303939346532323931306533323132643862653265346137353738 -63366238323130656430313831326330623961383237346436633165366634356466636162646261 -30303962386532353739643464653139386336393737316266653764373738373530323863326233 -36383135366336333831383438323261636665666331663338613266666338623630363533366338 -62626231306135643830336235353338396336646238396633363561326163623132653638306131 -39356634313130373862356263393364623337363136626361396536616631326233343730643864 -63633633343662343833333064356432373235653864303863656163643830333339656265646639 -65623636333630643236316434326264356435393837626330383962336161633262356638393266 -66323337346434333563323032393563663831653437336634613665306331366662623530393263 -34313033303966633235323339356461356630636238313031633834383434343035353438383035 -32386462323232623861626530373663333564333932616563613432383134393661303264303130 -30353431353062373533303237653966373762346264613734643730663932643633353631666565 -33633337633561343733623434653862383364646434663333366334323739653933313537363366 -65396532346439653334653739376562343136346266333739396630373166613763313563396133 -30303530326465653339343035623333646238393034323862623765633139396364346339326233 -36656664303536623834326662666537343537643336366138383935343065313538613363373966 -35393961356531623262363163343034656130383235373937643639613331356333323964633966 -65343632303237653939393438636137316332313931353934313034326366623832353363636564 -66363437616663356431613063613535666336383730333337623439653130653439663830633534 -39356266346231323535646265646439326137346361656632323564313435626165643231653735 -62636530383961343662393232666664363034656534663762393262356163336337373862316331 -62346535636464633363663330323335386230633465613965613630613135366131313138653061 -36613930653430643236323162643864353863323264323731333037306433376236306138303032 -65333963333364326665616333353031323761653764316362336337386230633736363766333662 -64316662343265623064643031326534346639353566376661343462613537373862303233316538 -64323364366430326264366232666263396235363736366437346438656265386464323264653733 -63363263636233353761356331613330353535346336366437613837343061663362663031333964 -34396366313363383863303035633934376432393334663364313866396166623563646530663632 -64613336653635636564373632646337386438333431326264373766306564326430326237363964 -65386631393135313566656435353039363638393535613063393730316634313837623630323161 -36356331323165613537386164343831386633663936313537306639633262633239353830313530 -31363066316332373035656134333937373263633561616431323334336335656131383436626464 -64356634626636356331383962643735303335363863343561373736363134646536393431346432 -62373434323665343533636237363461303335663632326332633535303139393433623032613465 -66383137613531376333323431356566653462616330656532633361623338343637613239303433 -62626537373165666562646561373731383436353436323434653562396334313639616661643935 -30616431313936626637353966366531316463616366306335656664613366303665346263633830 -31303939623631636463366132613866343032633365326433356463643832353230653339646161 -34636637373464386162623961373933363863346637313535643134613265383166393634656131 -64343930643162613362386465306465333235663461303566366266303664316565396563333433 -61356130333833653330666633346663373634396634383030346138313938636134303666623962 -62356636633062646639323963653565386464373736626161633565303335336262383230363732 -39393539333262326234333537356461663033303933383237346235373734656536303862333236 -65643563343932333632386635346438323339343463323338383865343362333430346338633932 -32386139343866633139313533663832356633633663616261386364373065373365386334383932 -63663334616532316133353565643936643035346464643936643862333466363166333066326338 -30326462333463393834303836653437353033313736643365313739313334383030636535656434 -32373231366462643233363837373133366139366130393537363361346133653839396432343936 -34303030373638303934343463326365663466633963323033323066383734336331353864313564 -64393638353735646233326638346464336637373465663562336661363834343138376635323363 -65356537356137336563383866356665383132366136633035646363393933643165656236313863 -30386363393466636435643666663533623332656232393537656161613837663064353838653835 -66666164386566663536653833613062333432363966303066323563363664333464633038353561 -32626634643264373132323638643766386362613639303262633462663737663337393631303436 -62326438396235666338313033383835306566366365396331653633323335333865366565343036 -62313835333964366537323565653662323563383663323033623939346533656665303235313264 -32623163383530376233353738633062346530376161353664333166323538666465363237616166 -34633561636234623263663535333939306239303334363939616636343334383838643939636664 -63623331306332386634646533396232323063336166613339636533353337306531636135313465 -39636164323537643230353234303666313161633366633831356530633635636561373461643136 -66323734386239656635633365373734643038636566626666646434663061333530633837633166 -38663633393764346565316639323638623039326332646432666232656463626335313737623862 -66386439616635633061356461343736656138633064653733643737636361626366313733356665 -34346565613330663938333563623436323934353934393235356234613766633235316137376632 -32386333373235393837393035653538303433373733386562623037363837396431393531303839 -31633162386637333963633335383662623661336133653866653336376366363665623165633032 -65636261616664326661333930343862663766393035613164313837333561326330333930643166 -38393161316364306139376234643363346564613463613536613534373936653532353861633139 -39663130316130366333303834386162343336346664316439373839333465356564303062633434 -62626337616366303730663661383863653834623537333933616239386264326634656235363166 -63323061343639666135306437356665313466326339356230376234363964653631643561633236 -31386339343033353062383231666135303338356335366335663763383237626138633961393239 -66363539333161313138626332303662366336323266643062663762323561656634636539336265 -34653736306231356135626435653736366133386538643338346463653231653333306633666532 -61323266303764653734643262363564656233653135623234663232353762653766313565613736 -31613432363333366233323165616561646366643634376639343966373635343335343661636462 -32613563613139363932636330663764353164316231333461653965373635363337633662653962 -35346463643363376633356234626464666332646331643132643138346665393937 +32356366663237323730376532393462356237346236616131303336623262653236356331333330 +3035633139313236643830343031323136363663383338350a613731626164323234656637653630 +61613839303334303866613762346363373731333962323066666132313664616438373365613635 +3139633033346437390a643430646134376364393737343234386661353938623531353937666430 +65366364613362393262313863346432306265353264353464313335646365326233376631373732 +63666661363462663633633461323635633065613633636664383233353661643638316662386633 +61626635386463323230366565386163393063383436636666643037353337343339373035366539 +31353561386565383232306263333231326261633132303036666562326635313365346633636634 +32353739633631666134373963383865363935326130336336333739393130353831646632623562 +61373038376462363864333061626564623239396135626537366439373063313039313966363439 +39363837356434356166353839363035373233656665663838346563623433346333643038323130 +35333239313863303066366265373365636535313339616238343435613737356635643833633066 +39613835306532383930303130633535666238323864386466336131313637336361313137306161 +61653063663131666539363463326332383763643064653133343633633066366362373235663736 +65376239316364666161323661613661333265366432366137386538646333653166646538626364 +30636263336235336363646136663266376538646631323665636561313537393834336565313033 +65663339626638303365333438393261653432373438373065343862373339623361383530343232 +64333964643837333666663434356534653561313864383137303761346364633461653635656566 +32613434373365616132353061333864333063386335653638623165333765653230313939353037 +38633966323032623937376565383134383034376539383632393064666464313439633330666335 +34313038333366353433646236666265366235373439626162306435663364643333316562346439 +32653466613966303933633538366466636461373930353364323030646436316436383562646664 +62626230363830306366613530633363653265623531383936396539356132393634323434356437 +66636661626335323737643538343834303238346631666334393934343134366331346532396137 +39396662396437656138626230343438336637663163303735653335663134343862373862386465 +63643362643462663961663165306533623931313133636365616665333537643365623833303439 +30656338323062366263636666333636356565653338323366346431396434633261623833376635 +62346563373163663163643036383730663939613032663161396432653039373561663264646135 +36303836656563333431633565383130616335356539333934313065333638363739393930393232 +65366537623836356361393865396534376238663264643834333230636636666139643734336161 +63333164303132373136623834666566663864336331666533646130393465613039613166306135 +63373334346134396566656233393666333063313435626561356464363064643966373561346465 +30346334383766623539386338363363343938343166633834326261396637333963656437663137 +65633233383138666538663933303337623130613166373632623334386236313765633037613165 +65653565623264623965303133653938383261636163373238353632313365613264643834656330 +34383965666337663334333362623266356666643565626432383861366131336564353732356634 +62316164663463326539316264323433346430363431383131393832383530333634643065376536 +32616132626332623132333061353961396331366666326631303165303261326465303062323863 +34656164613132616138383537336466353461363233393161366566356664336331313661383835 +38393938633537313731626132376636343234336431393939643438363634326434333236333335 +36356538653465616663633337626439366132383434326539663434333133356364653133336639 +33313331373761643662313431386364303262383566333462323532343135376666333363333432 +61656234643666343531646665336534343533346530326265643335383962376662326631653638 +32626439303562623364353263316535646134363736323133393832653536353262306334393439 +32306531623134656438613738643562663339336438353537343837393164386338363263323762 +63333162316365323964313663383863363836643636326237626432623738323063303639373639 +65363330353736333831333431393332616264353263316338383165303237343037396362346265 +62663330643964663833346561363661613137646563396164383236633135343136653338353664 +35616164646332333338626336376138303133323335353031333566613732386335386365626635 +62353138343165346437666233633736646332313132396462633235323563636338333662636466 +37343364343533623034653165633263343238666265636365636162646262383834613238366635 +62656233653162346530393236343437623137313766306466646136643962343062643732306164 +38633432363365613864376436393566363239343761623838623232613033363438656136633231 +39303366623764616137646136353336343765353266363161343638303532356261666166616138 +64373865303035643866666335356639656134353339383666373332363631393030346433316466 +39663535393238613537343764316634633632616465343539643938313134653437386132336166 +32636365353161313030393830653565623933313165323661376635643638646237313661336566 +31323533353536306561663735383639353539663835366632333232343038646432666438326263 +35386162663264396562613338386336623536306530366266346336643763656532653733366534 +64393436313962356233323164383639363435656639303563393336623735613363373933313139 +37353930396531633531336333656339336264353864396461393162316365343261653939313039 +61653164646466613562386566663031356363383836633661373566386235626339646139393234 +61323237383131386531616463663465613731306636356230306530623261613838633562333031 +61326131393631633161386363386635633465613430616336356537616664383763333566363033 +35363634643537646337363639333039343761363032633934633339633765643330613035633939 +38353133646461653038383837656431306239306366323866623539383561316432626135386234 +63353533336330643263346338353538383663656161616162363538323061613237396332303866 +35326466663762616138373839383865316131346438353038626134313861613139336162616138 +33393665396439323738613030366439323136653164616163643433343438373131393666633638 +36333236323434376533626435353264663930663130343037616462306663616139363565313732 +32666166643830303536633439353130303235323166626634313139613565323235666131303432 +33316531623736333832356563383234346338663634366436306533366464366663646137396566 +31393034366439646161643361663436383437623839323366616537353436366366363036373565 +37396439633238313266383635623665396636333431313734303435323134356163656462323964 +39313335356133623234393536346539333231373035396233343832633566643337656237623465 +35623764343934646261333266613562363630396532653764303531383666363766383765363734 +37663537613633643830326237363335623566376639633065363330313462633566653832643866 +35633434303665656462653438313838636638333734376539383931396535333837393330333739 +34313434653366663737616331626136626634646333303637363563336466303938363630333334 +30656264633065333137643865633030643230636363396565383530653734653134303032376231 +30643834636361313131653266303633626565666234336638386538633739376238343831343035 +35373265343966666462633261323538633531653238346466616234313537323938316239383162 +35326438313065313534313534626535313738323035386163663666653339613166373837346532 +65623235353136323961623631636437656234633337653036353461313939373464383431343163 +39333836336462636562396161303636666238663432343838383833306130633639663262616361 +30323331386363373262356666303036343836343636353865613965643365396238313564333966 +36363031386637313230303536303232613331666334393535363365653733353965396436373561 +63636137313836346164383239363862353830333038363562396133343536633335633235333133 +32636263633635366664363235666330633364303531376166666530646336346664323338636262 +39643534353535323534366164633435343963333331353233633432623738313038326132363962 +63316563376163386139656566656661396664666231373433613337326238303362383563623661 +63353762336336303362383132643766663138313739366634666531616363616161613836666665 +31396265353033663166366137393030366265333232663161356636343863333633646166313466 +61636238613366623835383633346538306130626132346239346166653736343138613633373765 +39623437623531313138313731313934623837313039363233653930646336393934373361303834 +66373464323066303735663862383266346131323365313932356132633037366337373136313434 +39373832346264363633306535623733363666393032353031373336353239646461306162653534 +63323132626362316538323264316362656233346130326165333334333834643333636234626530 +35633531663034636637373762333632336262373066356530613631343664633764643765613131 +66353036356664646264336131323539633038373438623261313335663763386334336662376634 +35376661316333363539633239653337663064613033363931653362323633386664366138363137 +30336536343430333936656536663331656631613733396632343764363163313862373936323663 +36306432663539336531663738363365376431616262346664613134366664373536386433333566 +31346133643639613934383964656162623037373462633031353561623238636230633731323536 +31326562343339376664653065656635353662663634386263363330356566336661313536666265 +66666165356534306163326137336331306430336536663965323964656233396638636563356365 +31366161363366656463373063636433343039303037646533626633613362393639323736623437 +30386665326534623266633463643535356633303639346632666633363464306230653837333937 +32636538353431633963366132366331323838393639613535613237646466666532386535323966 +38323363353764313332336137653364343232323164353733646631343565316133356436343936 +35636132663964633135363335383062663464346330666164396330616435663663363664643064 +38306162616232633239313565383836356566376238303164393539666265313066346632303165 +62393866346363653831623063346631396631653863303038373239643964666238333461303734 +33343136333535616332633038646663623232663730376438386538633936323033356534356532 +30313638326633396261343937333464386135646231653262646162303739613232326139303034 +39326633383937313536636466373561393866303664653264353033363039613664656637656539 +32616230663138323964663236363763643833373161313066363434636334636337323035643131 +63636131663537313438316632376134313666393437303031356662626531653639303363346335 +36393862366433343365643738666132666130303331303837306432623731383863613230373130 +65623030366661653439303561356533336465656636356164646636373765336239353461636566 +35616335643430663666653363666431356436343439643464356635313764663335613335396161 +62393661353733336535633436626366653031323663383133383930383034313236353362376637 +35323438633761383366383066363663373563656465316134373162396130616463333634376431 +32323465356535653938303732663464653966613834313463616161343637306236393266656565 +64666465313665633064343639656532633331663362366434393566643630333732346465626237 +38373433623037396364663165623263326262323962616636613865393536626661356133613764 +66633036623335646166356537313337353634326239646262303964633836636463343735313966 +34386335393564613961623231336638313835613935316335306438343565333330303866646636 +31663731663762643136323062643433636666626330376364396230663765613466373336303730 +65326434393465336532353236366139633935666130653735346265643463366563306266666537 +34636662386238343032616263363736643435313839393038643934313263613635366537623666 +33623034386166663831616563343733653866326263396261636635363337663239366261623538 +31363132356166363664653466626238636565633265346631626632356337303631613432346435 +39666137653636393135306135313465313035656538336434393233663361326139643237626439 +39613461333766333939636461323430623137383434343563333734336236346336626337313736 +31396535633136623333336465366166376338376235623537366636636331653737346165656364 +62663831323530626133323138393231306534323166643430656237336537306231623131383839 +65306661326533326539646266363461373461646464366264393364333232653535333934623738 +66343266333530613032316633323662356339366635373166623833646130643337613436363537 +39353737303862666234323230303137623434633636303263653762393937613234383862313561 +39613138356237373639663565653664643265373133356436343332363961616438643937393735 +31653461353030626664393137343431663264636263326535646665373130313961316639636631 +33626533336362356631366133636666343236396263626133626132343762633163633339313663 +65643930383164353931646538653065616362636335333236353764643864613338623465633066 +31333135633963383361623533316636346434316137633235343134663266373234383664323539 +31306233303234336130346665356133326431306430366364636662663630663833303664356632 +36633263653435333261313065653634653736393837306633393936386561613838343736326331 +65613233343866313835313733613630363736613064366138326364636566343961663338376231 +30636538303030303038353637663732336136656263303737623664376165393861643831616464 +37643433316634336238333432643039643334366632326234646339356333663739633934323632 +36626230653437303762343039643235646432663831363338643466396439376537353964663464 +31353433363037363135316535643336653430383333633465376631653162373630323636636365 +35376531633239393636396363626432306565656432653938396535663739386239 diff --git a/ansible/inventory/host_vars/asxpio-ops.yaml b/ansible/inventory/host_vars/asxpio-ops.yaml index 18a613a..7aa3f06 100644 --- a/ansible/inventory/host_vars/asxpio-ops.yaml +++ b/ansible/inventory/host_vars/asxpio-ops.yaml @@ -10,3 +10,22 @@ ssh_git_user: git # s3fs is off here: its ensure_diskfree reserve never fit the 38 GB disk s3fs_enabled: false s3fs_ensure_diskfree: 20000 + +# restic: Postgres as a dump, Garage with a fresh metadata snapshot, +# Forgejo's SQLite through the online backup API +restic_paths: + - /etc + - /opt + - /root/backups/rclone-hetzner.sh + - /var/lib/docker/volumes/storage_garage_meta + - /var/lib/docker/volumes/storage_garage_data +restic_excludes: + - /opt/backups + - /opt/containerd +restic_pre_hooks: + - docker exec storage-postgres pg_dumpall -U postgres > "$STAGE/pg_dumpall.sql" + - docker exec storage-postgres pg_dump -U postgres -Fc asxpio > "$STAGE/asxpio.dump" + - docker exec storage-garage /garage meta snapshot --all + - sqlite3 /opt/forgejo/data/gitea/gitea.db ".backup '$STAGE/forgejo-gitea.db'" + - sqlite3 /opt/vhpbot-prod/bot.db ".backup '$STAGE/vhpbot-prod.db'" +restic_schedule: "*-*-* 03:30:00" diff --git a/ansible/inventory/host_vars/asxpio-services.yaml b/ansible/inventory/host_vars/asxpio-services.yaml index 02be751..50000e7 100644 --- a/ansible/inventory/host_vars/asxpio-services.yaml +++ b/ansible/inventory/host_vars/asxpio-services.yaml @@ -6,3 +6,21 @@ wireguard_ip: "{{ asxpio_services_wireguard_ip }}" ssh_allow_public_git: false s3fs_ensure_diskfree: 81920 # 80 GB free = max 80 GB cache + +# restic: ejabberd's Mnesia through its own backup command, its SQLite +# through the online backup API. The Paper world is copied live (RCON is +# off, so there is no save-off); its autosave keeps that close enough. +restic_paths: + - /etc + - /opt + - /var/lib/docker/volumes/xmpp_database + - /var/lib/docker/volumes/xmpp_tackygram +restic_excludes: + - /opt/containerd + - /opt/minecraft/cache + - /opt/minecraft/libraries + - /opt/minecraft/versions +restic_pre_hooks: + - docker exec xmpp-ejabberd-1 ejabberdctl backup /opt/ejabberd/database/nightly.backup + - sqlite3 /var/lib/docker/volumes/xmpp_database/_data/ejabberd.sqlite ".backup '$STAGE/ejabberd.sqlite'" +restic_schedule: "*-*-* 04:15:00" diff --git a/ansible/roles/restic_backup/defaults/main.yaml b/ansible/roles/restic_backup/defaults/main.yaml new file mode 100644 index 0000000..9fb0ef1 --- /dev/null +++ b/ansible/roles/restic_backup/defaults/main.yaml @@ -0,0 +1,36 @@ +--- +# Nightly restic backup of the host to Hetzner Object Storage, one +# repository per region, written independently: losing one region or one +# repository leaves the other complete. + +# renovate: datasource=github-releases depName=restic/restic +restic_version: "0.19.1" + +restic_password: "{{ vault_restic_password }}" +restic_s3_access_key: "{{ s3_access_key }}" +restic_s3_secret_key: "{{ s3_secret_key }}" + +restic_repositories: + hel1: "s3:https://hel1.your-objectstorage.com/{{ s3_bucket_backups_main }}/restic/{{ inventory_hostname }}" + fsn1: "s3:https://fsn1.your-objectstorage.com/{{ s3_bucket_backups_repl }}/restic/{{ inventory_hostname }}" + +# Dumps and snapshots taken by the hooks land here and are backed up too +restic_stage_dir: /var/backups/restic-stage + +restic_paths: [/etc, /opt] +restic_excludes: [] +# Shell lines run in order before the backup, with $STAGE set. A failing +# hook marks the run as failed but the backup still runs. +restic_pre_hooks: [] + +restic_keep: "--keep-daily 7 --keep-weekly 4 --keep-monthly 6" +restic_schedule: "*-*-* 03:30:00" +# Prune (rewrites pack files, so costs S3 requests) once a week only +restic_prune_weekday: Sun + +# Failure mail, through Forgejo's SMTP account +restic_alert_smtp_host: smtp.fastmail.com +restic_alert_smtp_port: 587 +restic_alert_smtp_user: "{{ vault_forgejo_smtp_user }}" +restic_alert_smtp_password: "{{ vault_forgejo_smtp_password }}" +restic_alert_recipient: "{{ vault_crawler_recipient }}" diff --git a/ansible/roles/restic_backup/files/restic-alert b/ansible/roles/restic_backup/files/restic-alert new file mode 100644 index 0000000..096bfd3 --- /dev/null +++ b/ansible/roles/restic_backup/files/restic-alert @@ -0,0 +1,20 @@ +#!/usr/bin/python3 +# Mail the tail of a failed unit's journal. Usage: restic-alert +import os, smtplib, socket, subprocess, sys +from email.message import EmailMessage + +unit = sys.argv[1] +env = dict(l.rstrip("\n").split("=", 1) for l in open("/etc/restic/env") if "=" in l) +log = subprocess.run(["journalctl", "-u", unit, "-n", "60", "--no-pager", "-o", "cat"], + capture_output=True, text=True).stdout + +msg = EmailMessage() +msg["Subject"] = f"[{socket.gethostname()}] {unit} failed" +msg["From"] = env["ALERT_SMTP_USER"] +msg["To"] = env["ALERT_RECIPIENT"] +msg.set_content(log) + +with smtplib.SMTP(env["ALERT_SMTP_HOST"], int(env["ALERT_SMTP_PORT"]), timeout=30) as s: + s.starttls() + s.login(env["ALERT_SMTP_USER"], env["ALERT_SMTP_PASSWORD"]) + s.send_message(msg) diff --git a/ansible/roles/restic_backup/files/restic-alert@.service b/ansible/roles/restic_backup/files/restic-alert@.service new file mode 100644 index 0000000..498970e --- /dev/null +++ b/ansible/roles/restic_backup/files/restic-alert@.service @@ -0,0 +1,6 @@ +[Unit] +Description=Mail the failure of %i + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/restic-alert %i diff --git a/ansible/roles/restic_backup/handlers/main.yaml b/ansible/roles/restic_backup/handlers/main.yaml new file mode 100644 index 0000000..c28484f --- /dev/null +++ b/ansible/roles/restic_backup/handlers/main.yaml @@ -0,0 +1,4 @@ +--- +- name: Reload systemd + ansible.builtin.systemd: + daemon_reload: true diff --git a/ansible/roles/restic_backup/tasks/main.yaml b/ansible/roles/restic_backup/tasks/main.yaml new file mode 100644 index 0000000..dc2048e --- /dev/null +++ b/ansible/roles/restic_backup/tasks/main.yaml @@ -0,0 +1,88 @@ +--- +- name: Install sqlite3 for consistent copies of live databases + ansible.builtin.apt: + name: [sqlite3, bzip2] + state: present + +- name: Check the installed restic version + ansible.builtin.command: /usr/local/bin/restic version + register: restic_installed + changed_when: false + failed_when: false + check_mode: false + +- name: Install restic {{ restic_version }} + when: ("restic " ~ restic_version ~ " ") not in (restic_installed.stdout | default('')) + block: + - name: Download the release + ansible.builtin.get_url: + url: "https://github.com/restic/restic/releases/download/v{{ restic_version }}/restic_{{ restic_version }}_linux_amd64.bz2" + dest: "/tmp/restic_{{ restic_version }}.bz2" + checksum: "sha256:https://github.com/restic/restic/releases/download/v{{ restic_version }}/SHA256SUMS" + mode: "0600" + + - name: Unpack it + ansible.builtin.shell: > + bunzip2 -c /tmp/restic_{{ restic_version }}.bz2 > /usr/local/bin/restic.new && + chmod 755 /usr/local/bin/restic.new && + mv /usr/local/bin/restic.new /usr/local/bin/restic && + rm /tmp/restic_{{ restic_version }}.bz2 + changed_when: true + +- name: Create config and cache directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + mode: "0700" + loop: [/etc/restic, /var/cache/restic] + +- name: Write credentials + ansible.builtin.template: + src: env.j2 + dest: /etc/restic/env + mode: "0600" + diff: false + +- name: Install the scripts + ansible.builtin.template: + src: restic-backup.j2 + dest: /usr/local/sbin/restic-backup + mode: "0700" + +- name: Install the alert script + ansible.builtin.copy: + src: restic-alert + dest: /usr/local/sbin/restic-alert + mode: "0700" + +- name: Install the systemd units + ansible.builtin.template: + src: "{{ item }}.j2" + dest: "/etc/systemd/system/{{ item }}" + mode: "0644" + loop: [restic-backup.service, restic-backup.timer] + notify: Reload systemd + +- name: Install the alert unit + ansible.builtin.copy: + src: restic-alert@.service + dest: /etc/systemd/system/restic-alert@.service + mode: "0644" + notify: Reload systemd + +- name: Flush handlers + ansible.builtin.meta: flush_handlers + +- name: Create the repositories + ansible.builtin.command: /usr/local/sbin/restic-backup init + register: restic_init + changed_when: "'created' in restic_init.stdout" + when: not ansible_check_mode + +- name: Enable the nightly timer + ansible.builtin.systemd: + name: restic-backup.timer + enabled: true + state: started + # On a first dry run the unit is not installed yet + ignore_errors: "{{ ansible_check_mode }}" diff --git a/ansible/roles/restic_backup/templates/env.j2 b/ansible/roles/restic_backup/templates/env.j2 new file mode 100644 index 0000000..9b36af8 --- /dev/null +++ b/ansible/roles/restic_backup/templates/env.j2 @@ -0,0 +1,8 @@ +RESTIC_PASSWORD={{ restic_password }} +AWS_ACCESS_KEY_ID={{ restic_s3_access_key }} +AWS_SECRET_ACCESS_KEY={{ restic_s3_secret_key }} +ALERT_SMTP_HOST={{ restic_alert_smtp_host }} +ALERT_SMTP_PORT={{ restic_alert_smtp_port }} +ALERT_SMTP_USER={{ restic_alert_smtp_user }} +ALERT_SMTP_PASSWORD={{ restic_alert_smtp_password }} +ALERT_RECIPIENT={{ restic_alert_recipient }} diff --git a/ansible/roles/restic_backup/templates/restic-backup.j2 b/ansible/roles/restic_backup/templates/restic-backup.j2 new file mode 100644 index 0000000..e0d10aa --- /dev/null +++ b/ansible/roles/restic_backup/templates/restic-backup.j2 @@ -0,0 +1,70 @@ +#!/bin/bash +# Managed by Ansible (role restic_backup). +# +# restic-backup run hooks, then backup + forget to every repository +# restic-backup init create the repositories that do not exist yet +# restic-backup plain restic against one repository, e.g. +# restic-backup hel1 snapshots +# restic-backup fsn1 restore latest --target /tmp/r +# +# Repositories: {{ restic_repositories.keys() | join(', ') }} +set -uo pipefail + +set -a; . /etc/restic/env; set +a +export RESTIC_CACHE_DIR=/var/cache/restic +STAGE={{ restic_stage_dir }} + +declare -A REPOS=( +{% for name, url in restic_repositories.items() %} + [{{ name }}]="{{ url }}" +{% endfor %} +) + +log() { echo "restic-backup: $*"; } + +case "${1:-}" in +init) + for name in "${!REPOS[@]}"; do + if RESTIC_REPOSITORY="${REPOS[$name]}" restic cat config >/dev/null 2>&1; then + log "$name: exists" + else + RESTIC_REPOSITORY="${REPOS[$name]}" restic init || exit 1 + log "$name: created" + fi + done + ;; + +run) + failed=0 + rm -rf "$STAGE"; install -d -m 700 "$STAGE" + export STAGE +{% for hook in restic_pre_hooks %} + log "hook {{ loop.index }}/{{ restic_pre_hooks | length }}" + ( set -e; {{ hook }} ) || { log "hook failed"; failed=1; } +{% endfor %} + + for name in "${!REPOS[@]}"; do + export RESTIC_REPOSITORY="${REPOS[$name]}" + log "$name: backup" + restic backup --host {{ inventory_hostname }} --one-file-system --exclude-caches \ +{% for e in restic_excludes %} + --exclude '{{ e }}' \ +{% endfor %} + {{ (restic_paths + [restic_stage_dir]) | join(' ') }} || { failed=1; continue; } + + prune="" + [ "$(date +%a)" = "{{ restic_prune_weekday }}" ] && prune="--prune" + log "$name: forget $prune" + restic forget --host {{ inventory_hostname }} {{ restic_keep }} $prune || failed=1 + done + + rm -rf "$STAGE" + exit $failed + ;; + +*) + [ -n "${1:-}" ] && [ -n "${REPOS[$1]:-}" ] || { sed -n '4,10p' "$0"; exit 2; } + export RESTIC_REPOSITORY="${REPOS[$1]}"; shift + exec restic "$@" + ;; +esac diff --git a/ansible/roles/restic_backup/templates/restic-backup.service.j2 b/ansible/roles/restic_backup/templates/restic-backup.service.j2 new file mode 100644 index 0000000..b7d2384 --- /dev/null +++ b/ansible/roles/restic_backup/templates/restic-backup.service.j2 @@ -0,0 +1,12 @@ +[Unit] +Description=restic backup to Hetzner Object Storage ({{ restic_repositories.keys() | join(', ') }}) +Wants=network-online.target +After=network-online.target docker.service +OnFailure=restic-alert@%n.service + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/restic-backup run +Nice=10 +IOSchedulingClass=idle +TimeoutStartSec=6h diff --git a/ansible/roles/restic_backup/templates/restic-backup.timer.j2 b/ansible/roles/restic_backup/templates/restic-backup.timer.j2 new file mode 100644 index 0000000..42fae04 --- /dev/null +++ b/ansible/roles/restic_backup/templates/restic-backup.timer.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Nightly restic backup + +[Timer] +OnCalendar={{ restic_schedule }} +RandomizedDelaySec=10m +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/site.yaml b/ansible/site.yaml index f1a81ca..f5f7ffa 100644 --- a/ansible/site.yaml +++ b/ansible/site.yaml @@ -36,6 +36,12 @@ roles: - { role: forgejo, tags: [forgejo] } +- name: Backups + hosts: asxpio + become: true + roles: + - { role: restic_backup, tags: [restic] } + # Last: when the pipeline changes the runner it recreates the container # that is running the pipeline. Everything else is done by then; that job # ends as failed and the next run confirms the result.