Add Garage beside MinIO

MinIO is abandoned upstream and its images are gone from the registries,
so the object store cannot be rebuilt or updated. Garage takes its
place.

This is the first step: Garage runs as one node next to MinIO, with the
same three buckets and one key per app, and is not routed yet. MinIO
keeps serving s3.asxp.io and is no longer bootstrapped, so its users
stay as they are until the data is copied and the apps are switched.

Garage only accepts keys in its own format. The bootstrap hashes the
existing secrets into that format, as the apps' deploys will, so no
secret has to change for the move; a secret already in Garage's format
is used as it is.

Garage has no anonymous S3 access. The public bucket is marked as a
website instead, to be served through Garage's web endpoint.

Assisted-by: Claude:fable-5-2

.forgejo/workflows/deploy.yaml

@@ -23,7 +23,7 @@ jobs:
           REMOTE_HOST: ${{ secrets.DEPLOY_IP }}
           REMOTE_USER: ${{ secrets.DEPLOY_USER }}
           ARGS: "-rltgoDzvO --delete"
-          SOURCE: "docker-compose.yml init"
+          SOURCE: "docker-compose.yml garage.toml init"
           TARGET: "/opt/storage/"
 
       - name: Write .env from secrets and bring stack up
@@ -63,15 +63,26 @@ jobs:
             PUBLIC_S3_SECRET_KEY=${PUBLIC_S3_SECRET_KEY}
             EOF
             chmod 600 /opt/storage/.env
-            chmod +x /opt/storage/init/postgres-init.sh /opt/storage/init/minio-bootstrap.sh
+            chmod +x /opt/storage/init/postgres-init.sh /opt/storage/init/garage-bootstrap.sh
+
+            # Garage's RPC secret never leaves the host: made once, kept
+            # outside the rsynced paths.
+            mkdir -p /opt/storage/secrets
+            if [ ! -s /opt/storage/secrets/garage_rpc_secret ]; then
+              head -c 32 /dev/urandom | od -An -tx1 | tr -d ' \n' > /opt/storage/secrets/garage_rpc_secret
+            fi
+            chmod 600 /opt/storage/secrets/garage_rpc_secret
 
             cd /opt/storage
             # minio/minio and minio/mc are gone from Docker Hub (upstream
             # abandoned); the host's cached copies are all there is.
             docker compose pull --ignore-pull-failures
-            docker compose up -d postgres minio
+            docker compose up -d postgres minio garage
-            # Run init containers (idempotent) and wait for them to exit.
+            # Run the init steps (idempotent).
             docker compose up --exit-code-from postgres-init postgres-init
-            docker compose up --exit-code-from minio-init minio-init
+            ASXPIO_S3_ACCESS_KEY="${ASXPIO_S3_ACCESS_KEY}" ASXPIO_S3_SECRET_KEY="${ASXPIO_S3_SECRET_KEY}" \
+            BINGO_S3_ACCESS_KEY="${BINGO_S3_ACCESS_KEY}" BINGO_S3_SECRET_KEY="${BINGO_S3_SECRET_KEY}" \
+            PUBLIC_S3_ACCESS_KEY="${PUBLIC_S3_ACCESS_KEY}" PUBLIC_S3_SECRET_KEY="${PUBLIC_S3_SECRET_KEY}" \
+              sh init/garage-bootstrap.sh
 
             echo "Storage stack deployed."

docker-compose.yml

@@ -65,31 +65,35 @@ services:
       - "traefik.http.routers.minio-s3.service=minio-s3"
       - "traefik.http.services.minio-s3.loadbalancer.server.port=9000"
 
-  minio-init:
+  # Replaces MinIO (abandoned upstream; its images are gone from the
-    image: minio/mc:latest
+  # registries). Not routed yet: MinIO still answers s3.asxp.io until the data
-    container_name: storage-minio-init
+  # is copied and the apps are switched.
-    depends_on:
+  garage:
-      minio:
+    image: dxflrs/garage:v2.5.0
-        condition: service_healthy
+    container_name: storage-garage
-    environment:
+    restart: unless-stopped
-      MINIO_ROOT_USER: ${MINIO_ROOT_USER}
+    command: ["/garage", "server", "--single-node"]
-      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
-      ASXPIO_S3_ACCESS_KEY: ${ASXPIO_S3_ACCESS_KEY}
-      ASXPIO_S3_SECRET_KEY: ${ASXPIO_S3_SECRET_KEY}
-      BINGO_S3_ACCESS_KEY: ${BINGO_S3_ACCESS_KEY}
-      BINGO_S3_SECRET_KEY: ${BINGO_S3_SECRET_KEY}
-      PUBLIC_S3_ACCESS_KEY: ${PUBLIC_S3_ACCESS_KEY}
-      PUBLIC_S3_SECRET_KEY: ${PUBLIC_S3_SECRET_KEY}
     volumes:
-      - ./init/minio-bootstrap.sh:/bootstrap.sh:ro
+      - ./garage.toml:/etc/garage.toml:ro
-    entrypoint: ["/bin/sh", "/bootstrap.sh"]
+      - ./secrets/garage_rpc_secret:/run/secrets/garage_rpc_secret:ro
+      - garage_meta:/var/lib/garage/meta
+      - garage_data:/var/lib/garage/data
     networks:
       - storage
-    restart: "no"
+      - traefik
+    healthcheck:
+      test: ["CMD", "/garage", "status"]
+      interval: 10s
+      timeout: 5s
+      retries: 10
+    labels:
+      - "traefik.enable=false"
 
 volumes:
   pg_data:
   minio_data:
+  garage_meta:
+  garage_data:
 
 networks:
   storage:

garage.toml

@@ -0,0 +1,30 @@
+# Garage, single node. No secrets here: the RPC secret is a file made on the
+# host at first deploy (see the deploy workflow).
+metadata_dir = "/var/lib/garage/meta"
+data_dir = "/var/lib/garage/data"
+
+# One node and no replica to repair from, so prefer the engine that survives
+# an unclean stop, sync writes, and keep metadata snapshots.
+db_engine = "sqlite"
+metadata_fsync = true
+data_fsync = true
+metadata_auto_snapshot_interval = "6h"
+
+replication_factor = 1
+
+rpc_bind_addr = "[::]:3901"
+rpc_public_addr = "127.0.0.1:3901"
+rpc_secret_file = "/run/secrets/garage_rpc_secret"
+
+[s3_api]
+# The apps sign for us-east-1, as they did against MinIO.
+s3_region = "us-east-1"
+api_bind_addr = "[::]:3900"
+
+# Anonymous reads. Garage has no anonymous S3 access; a bucket marked as a
+# website is served here, picked by Host: <bucket>.web.garage. Traefik sends
+# https://s3.asxp.io/public/<key> this way.
+[s3_web]
+bind_addr = "[::]:3902"
+root_domain = ".web.garage"
+index = "index.html"

init/garage-bootstrap.sh

@@ -0,0 +1,103 @@
+#!/bin/sh
+# Idempotent Garage bootstrap, run on the host after `docker compose up`:
+# buckets, per-app keys and what each key may do. The Garage image has no
+# shell, so this drives its CLI through `docker exec`.
+#
+# Env: ASXPIO_S3_ACCESS_KEY/_SECRET_KEY (required), BINGO_S3_* and
+# PUBLIC_S3_* (optional). GARAGE_CONTAINER overrides the container name.
+set -eu
+
+c="${GARAGE_CONTAINER:-storage-garage}"
+g() { docker exec "$c" /garage "$@"; }
+
+# Garage only takes its own key format: an id of GK + 24 hex digits and a
+# secret of 64 hex digits. A value already in that format is used as it is;
+# anything else (the MinIO-era secrets) is hashed into it, the same way the
+# apps' deploy workflows do, so the secrets did not have to change with the
+# move. Replace a secret with a native Garage pair whenever it is rotated.
+garage_key_id() {
+  if printf '%s' "$1" | grep -Eq '^GK[0-9a-f]{24}$'; then
+    printf '%s' "$1"
+  else
+    printf 'GK%s' "$(printf '%s' "$1" | sha256sum | cut -c1-24)"
+  fi
+}
+garage_secret() {
+  if printf '%s' "$1" | grep -Eq '^[0-9a-f]{64}$'; then
+    printf '%s' "$1"
+  else
+    printf '%s' "$1" | sha256sum | cut -c1-64
+  fi
+}
+
+i=0
+until g status >/dev/null 2>&1; do
+  i=$((i + 1))
+  if [ "$i" -gt 60 ]; then
+    echo "garage-init: garage did not come up" >&2
+    exit 1
+  fi
+  sleep 1
+done
+
+ensure_bucket() {
+  g bucket info "$1" >/dev/null 2>&1 || g bucket create "$1" >/dev/null
+}
+
+# $1 name, $2 access key, $3 secret key. Prints the Garage key id.
+ensure_key() {
+  id="$(garage_key_id "$2")"
+  secret="$(garage_secret "$3")"
+  if g key info --show-secret "$id" >/dev/null 2>&1; then
+    if ! g key info --show-secret "$id" | grep -q "$secret"; then
+      # Rotated secret under the same id.
+      g key delete --yes "$id" >/dev/null
+      g key import --yes -n "$1" "$id" "$secret" >/dev/null
+    fi
+  else
+    g key import --yes -n "$1" "$id" "$secret" >/dev/null
+  fi
+  printf '%s' "$id"
+}
+
+# $1 bucket, $2 key id, $3 "rw", "ro" or "wo". Sets exactly that.
+grant() {
+  case "$3" in
+    rw) g bucket allow --read --write "$1" --key "$2" >/dev/null
+        g bucket deny --owner "$1" --key "$2" >/dev/null ;;
+    ro) g bucket allow --read "$1" --key "$2" >/dev/null
+        g bucket deny --write --owner "$1" --key "$2" >/dev/null ;;
+    wo) g bucket allow --write "$1" --key "$2" >/dev/null
+        g bucket deny --read --owner "$1" --key "$2" >/dev/null ;;
+  esac
+}
+
+ensure_bucket "asxpio-invoices"
+# Datacenter bingo photos: uploaded and resized by asxpio's admin, served by
+# datacenterbingo.
+ensure_bucket "datacenterbingo-photos"
+# Files shared by link: https://s3.asxp.io/public/<key>. Everything in this
+# bucket is readable by anyone who knows the key; nobody can list it.
+ensure_bucket "public"
+g bucket website --allow public >/dev/null
+
+asxpio="$(ensure_key asxpio "${ASXPIO_S3_ACCESS_KEY}" "${ASXPIO_S3_SECRET_KEY}")"
+grant "asxpio-invoices" "$asxpio" rw
+grant "datacenterbingo-photos" "$asxpio" rw
+
+if [ -n "${BINGO_S3_ACCESS_KEY:-}" ] && [ -n "${BINGO_S3_SECRET_KEY:-}" ]; then
+  bingo="$(ensure_key bingo "${BINGO_S3_ACCESS_KEY}" "${BINGO_S3_SECRET_KEY}")"
+  grant "datacenterbingo-photos" "$bingo" ro
+else
+  echo "garage-init: BINGO_S3_* unset, skipping bingo key"
+fi
+
+# Put and delete only: the holder can publish and retract, not read or list.
+if [ -n "${PUBLIC_S3_ACCESS_KEY:-}" ] && [ -n "${PUBLIC_S3_SECRET_KEY:-}" ]; then
+  uploader="$(ensure_key public-upload "${PUBLIC_S3_ACCESS_KEY}" "${PUBLIC_S3_SECRET_KEY}")"
+  grant "public" "$uploader" wo
+else
+  echo "garage-init: PUBLIC_S3_* unset, skipping public uploader"
+fi
+
+echo "garage-init: done"