iac/nixos/modules/backup.nix

# Nightly restic backup to Hetzner Object Storage, one repository per
# region, written independently: losing one region or one repository
# leaves the other complete. The same script and command as the Ansible
# role restic_backup installs on asxpio-ops:
#
#   restic-backup hel1 snapshots
#   restic-backup fsn1 restore latest --target /tmp/r --include /opt/xmpp
{
  config,
  lib,
  pkgs,
  ...
}:
let
  cfg = config.asxpio.backup;
  host = config.networking.hostName;

  # The release the Ansible role pins (restic_version), so both hosts
  # write their repositories with the same version. nixpkgs 26.05 has an
  # older one. To bump: change the version, then take the new hash from
  #   nix store prefetch-file <url>
  resticVersion = "0.19.1";
  restic =
    pkgs.runCommand "restic-${resticVersion}"
      {
        src = pkgs.fetchurl {
          url = "https://github.com/restic/restic/releases/download/v${resticVersion}/restic_${resticVersion}_linux_amd64.bz2";
          hash = "sha256-9BVBViTcxFLyoCuMM2QXkajG1tO2W7s1Q/z5olFRWFw=";
        };
        nativeBuildInputs = [ pkgs.bzip2 ];
      }
      ''
        mkdir -p $out/bin
        bunzip2 -c $src > $out/bin/restic
        chmod 755 $out/bin/restic
      '';

  repositories = {
    hel1 = "s3:https://hel1.your-objectstorage.com/asxpio-backups-main/restic/${host}";
    fsn1 = "s3:https://fsn1.your-objectstorage.com/asxpio-backups-repl/restic/${host}";
  };

  envFile = config.sops.templates."restic-env".path;
  secret = name: config.sops.placeholder.${name};

  hooks = lib.imap1 (i: hook: ''
    log "hook ${toString i}/${toString (lib.length cfg.preHooks)}"
    ( set -e; ${hook} ) || { log "hook failed"; failed=1; }
  '') cfg.preHooks;

  backup = pkgs.writeShellApplication {
    name = "restic-backup";
    runtimeInputs = [
      restic
      pkgs.coreutils
    ]
    ++ cfg.hookPackages;
    # No errexit: a failing hook or repository must not stop the others
    bashOptions = [
      "nounset"
      "pipefail"
    ];
    text = ''
      usage() {
        cat <<'USAGE'
      restic-backup run                 hooks, then backup + forget to every repository
      restic-backup init                create the repositories that do not exist yet
      restic-backup <repo> <args...>    plain restic against one repository, e.g.
                                        restic-backup hel1 snapshots
                                        restic-backup fsn1 restore latest --target /tmp/r

      Repositories: ${lib.concatStringsSep ", " (lib.attrNames repositories)}
      USAGE
      }

      set -a
      # shellcheck source=/dev/null
      . ${envFile}
      set +a
      export RESTIC_CACHE_DIR=/var/cache/restic
      STAGE=${cfg.stageDir}

      declare -A REPOS=(
      ${lib.concatStrings (lib.mapAttrsToList (name: url: "  [${name}]=\"${url}\"\n") repositories)})

      log() { echo "restic-backup: $*"; }

      case "''${1:-}" in
      init)
        for name in "''${!REPOS[@]}"; do
          if RESTIC_REPOSITORY="''${REPOS[$name]}" restic cat config >/dev/null 2>&1; then
            log "$name: exists"
          else
            RESTIC_REPOSITORY="''${REPOS[$name]}" restic init || exit 1
            log "$name: created"
          fi
        done
        ;;

      run)
        failed=0
        rm -rf "$STAGE"
        install -d -m 700 "$STAGE"
        export STAGE
      ${lib.concatStrings hooks}
        for name in "''${!REPOS[@]}"; do
          export RESTIC_REPOSITORY="''${REPOS[$name]}"
          log "$name: backup"
          restic backup --host ${host} --one-file-system --exclude-caches \
            ${lib.concatMapStringsSep " " (e: "--exclude ${lib.escapeShellArg e}") cfg.excludes} \
            ${lib.escapeShellArgs cfg.paths} "$STAGE" || { failed=1; continue; }

          # Prune rewrites pack files, so costs S3 requests: once a week only
          prune=()
          [ "$(date +%a)" = Sun ] && prune=(--prune)
          log "$name: forget ''${prune[*]}"
          restic forget --host ${host} --keep-daily 7 --keep-weekly 4 --keep-monthly 6 \
            "''${prune[@]}" || failed=1
        done

        rm -rf "$STAGE"
        exit $failed
        ;;

      *)
        [ -n "''${1:-}" ] && [ -n "''${REPOS[$1]:-}" ] || { usage; exit 2; }
        export RESTIC_REPOSITORY="''${REPOS[$1]}"
        shift
        exec restic "$@"
        ;;
      esac
    '';
  };

  alert = pkgs.writers.writePython3Bin "restic-alert" { doCheck = false; } (
    builtins.readFile ./restic-alert.py
  );
in
{
  options.asxpio.backup = {
    paths = lib.mkOption {
      type = lib.types.listOf lib.types.str;
      default = [
        "/etc"
        "/opt"
      ];
      description = "What to back up, besides the stage directory.";
    };
    excludes = lib.mkOption {
      type = lib.types.listOf lib.types.str;
      default = [ ];
    };
    preHooks = lib.mkOption {
      type = lib.types.listOf lib.types.str;
      default = [ ];
      description = ''
        Shell lines run in order before the backup, with $STAGE set. A
        failing hook marks the run as failed but the backup still runs.
      '';
    };
    hookPackages = lib.mkOption {
      type = lib.types.listOf lib.types.package;
      default = [ ];
      description = "Programs the hooks call.";
    };
    stageDir = lib.mkOption {
      type = lib.types.str;
      default = "/var/backups/restic-stage";
      description = "Dumps written by the hooks land here and are backed up too.";
    };
    schedule = lib.mkOption {
      type = lib.types.str;
      default = "*-*-* 03:30:00";
    };
  };

  config = {
    sops.secrets = {
      "restic/password" = { };
      "s3/access_key" = { };
      "s3/secret_key" = { };
      "smtp/user" = { };
      "smtp/password" = { };
      "alert/recipient" = { };
    };
    sops.templates."restic-env".content = ''
      RESTIC_PASSWORD=${secret "restic/password"}
      AWS_ACCESS_KEY_ID=${secret "s3/access_key"}
      AWS_SECRET_ACCESS_KEY=${secret "s3/secret_key"}
      ALERT_SMTP_HOST=smtp.fastmail.com
      ALERT_SMTP_PORT=587
      ALERT_SMTP_USER=${secret "smtp/user"}
      ALERT_SMTP_PASSWORD=${secret "smtp/password"}
      ALERT_RECIPIENT=${secret "alert/recipient"}
    '';

    environment.systemPackages = [ backup ];
    systemd.tmpfiles.rules = [
      "d /var/cache/restic 0700 root root -"
      "d /var/backups 0755 root root -"
    ];

    systemd.services.restic-backup = {
      description = "restic backup to Hetzner Object Storage (${lib.concatStringsSep ", " (lib.attrNames repositories)})";
      wants = [ "network-online.target" ];
      after = [
        "network-online.target"
        "docker.service"
      ];
      onFailure = [ "restic-alert@%n.service" ];
      serviceConfig = {
        Type = "oneshot";
        ExecStart = "${lib.getExe backup} run";
        Nice = 10;
        IOSchedulingClass = "idle";
        TimeoutStartSec = "6h";
      };
    };
    systemd.timers.restic-backup = {
      description = "Nightly restic backup";
      wantedBy = [ "timers.target" ];
      timerConfig = {
        OnCalendar = cfg.schedule;
        RandomizedDelaySec = "10m";
        Persistent = true;
      };
    };

    # Mails the tail of a failed unit's journal, through Forgejo's SMTP
    # account. Any unit can use it: OnFailure=restic-alert@%n.service
    systemd.services."restic-alert@" = {
      description = "Mail the failure of %i";
      serviceConfig = {
        Type = "oneshot";
        ExecStart = "${lib.getExe alert} %i ${envFile}";
      };
    };
  };
}