From e9388c0b0d2d36e19373ae6366df80bb3b94d3a0 Mon Sep 17 00:00:00 2001
From: Sergei Poljanski <me@asxp.io>
Date: Wed, 11 Nov 2026 03:55:18 +0400
Subject: [PATCH] Remove MinIO

Garage serves all three buckets and both apps, and a last comparison
found nothing in MinIO that Garage lacks. Drop the MinIO service, its
bootstrap and its root credentials from the deploy, and let compose
remove the leftover containers.

The storage_minio_data volume is left on the host. All images can be
pulled again, so the deploy goes back to failing on a pull error.

Document Garage in the README and SECRETS, including the key format and
how the existing secrets map to it, and swap the MinIO client in the
flake for awscli and rclone.

Assisted-by: Claude:fable-5-2
---
 .forgejo/workflows/deploy.yaml |  12 +--
 README.md                      |  51 ++++++++----
 SECRETS.md                     |  63 +++++++-------
 docker-compose.yml             |  24 ------
 flake.nix                      |   7 +-
 init/minio-bootstrap.sh        | 148 ---------------------------------
 6 files changed, 74 insertions(+), 231 deletions(-)
 delete mode 100644 init/minio-bootstrap.sh

diff --git a/.forgejo/workflows/deploy.yaml b/.forgejo/workflows/deploy.yaml
index ace0d22..3b1157d 100644
--- a/.forgejo/workflows/deploy.yaml
+++ b/.forgejo/workflows/deploy.yaml
@@ -30,8 +30,6 @@ jobs:
         uses: appleboy/ssh-action@v1.2.3
         env:
           POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
-          MINIO_ROOT_USER: ${{ secrets.MINIO_ROOT_USER }}
-          MINIO_ROOT_PASSWORD: ${{ secrets.MINIO_ROOT_PASSWORD }}
           ASXPIO_DB_PASSWORD: ${{ secrets.ASXPIO_DB_PASSWORD }}
           ASXPIO_S3_ACCESS_KEY: ${{ secrets.ASXPIO_S3_ACCESS_KEY }}
           ASXPIO_S3_SECRET_KEY: ${{ secrets.ASXPIO_S3_SECRET_KEY }}
@@ -44,15 +42,13 @@ jobs:
           host: ${{ secrets.DEPLOY_IP }}
           username: ${{ secrets.DEPLOY_USER }}
           key: ${{ secrets.DEPLOY_SSH_KEY }}
-          envs: POSTGRES_PASSWORD,MINIO_ROOT_USER,MINIO_ROOT_PASSWORD,ASXPIO_DB_PASSWORD,ASXPIO_S3_ACCESS_KEY,ASXPIO_S3_SECRET_KEY,BINGO_DB_PASSWORD,BINGO_S3_ACCESS_KEY,BINGO_S3_SECRET_KEY,PUBLIC_S3_ACCESS_KEY,PUBLIC_S3_SECRET_KEY
+          envs: POSTGRES_PASSWORD,ASXPIO_DB_PASSWORD,ASXPIO_S3_ACCESS_KEY,ASXPIO_S3_SECRET_KEY,BINGO_DB_PASSWORD,BINGO_S3_ACCESS_KEY,BINGO_S3_SECRET_KEY,PUBLIC_S3_ACCESS_KEY,PUBLIC_S3_SECRET_KEY
           script_stop: true
           script: |
             umask 077
             mkdir -p /opt/storage
             cat > /opt/storage/.env <<EOF
             POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
-            MINIO_ROOT_USER=${MINIO_ROOT_USER}
-            MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD}
             ASXPIO_DB_PASSWORD=${ASXPIO_DB_PASSWORD}
             ASXPIO_S3_ACCESS_KEY=${ASXPIO_S3_ACCESS_KEY}
             ASXPIO_S3_SECRET_KEY=${ASXPIO_S3_SECRET_KEY}
@@ -74,10 +70,8 @@ jobs:
             chmod 600 /opt/storage/secrets/garage_rpc_secret
 
             cd /opt/storage
-            # minio/minio and minio/mc are gone from Docker Hub (upstream
-            # abandoned); the host's cached copies are all there is.
-            docker compose pull --ignore-pull-failures
-            docker compose up -d postgres minio garage
+            docker compose pull
+            docker compose up -d --remove-orphans postgres garage
             # Run the init steps (idempotent).
             docker compose up --exit-code-from postgres-init postgres-init
             ASXPIO_S3_ACCESS_KEY="${ASXPIO_S3_ACCESS_KEY}" ASXPIO_S3_SECRET_KEY="${ASXPIO_S3_SECRET_KEY}" \
diff --git a/README.md b/README.md
index bc59980..da8f14c 100644
--- a/README.md
+++ b/README.md
@@ -1,14 +1,16 @@
 # storage
 
-Postgres + MinIO for asxp.io and any future small services that need a database
+Postgres + Garage (S3) for asxp.io and any future small services that need a database
 or object storage. Single-host, no clustering. Deploys via Forgejo Actions to
 the same box that runs the apps.
 
 ## What's here
 
-- `docker-compose.yml` — `postgres`, `minio`, and one-shot init containers.
+- `docker-compose.yml` — `postgres`, its one-shot init container, and `garage`.
+- `garage.toml` — Garage's config (single node, no secrets in it).
 - `init/postgres-init.sh` — idempotent role/DB bootstrap.
-- `init/minio-bootstrap.sh` — idempotent bucket + per-app service account.
+- `init/garage-bootstrap.sh` — idempotent buckets, per-app keys and grants;
+  run on the host by the deploy, through `docker exec`.
 - `.forgejo/workflows/deploy.yaml` — pushes compose + `.env` to the prod host
   and runs `docker compose up -d`.
 
@@ -17,15 +19,29 @@ the same box that runs the apps.
 Two Docker networks:
 
 - `storage` — internal-only; apps join this as an external network to reach
-  `postgres:5432` and `minio:9000`.
-- `traefik` — existing reverse-proxy network. Only MinIO joins it, to expose
+  `postgres:5432` and `garage:3900`.
+- `traefik` — existing reverse-proxy network. Only Garage joins it, to expose
   the S3 API at `https://s3.asxp.io`.
 
-The MinIO console (`:9001`) is **not** exposed externally. To use it, SSH-port-forward:
+Garage replaced MinIO in November 2026 (MinIO was abandoned upstream and its
+images removed from the registries).
+
+## The `public` bucket
+
+Garage answers no anonymous S3 request. `public` is marked as a website
+instead, and Traefik sends unsigned GET/HEAD of
+`https://s3.asxp.io/public/<key>` to Garage's web endpoint. Anyone who knows a
+key can read it; nobody can list the bucket. Uploading needs the uploader key
+(see `SECRETS.md`).
+
+## Looking inside
+
+The image has no shell; use the CLI through `docker exec`:
 
 ```sh
-ssh -L 9001:storage-minio:9001 deploy@prod
-# then open http://localhost:9001
+ssh prod 'docker exec storage-garage /garage status'
+ssh prod 'docker exec storage-garage /garage bucket list'
+ssh prod 'docker exec storage-garage /garage key list'
 ```
 
 ## Adding a new app
@@ -35,20 +51,27 @@ ssh -L 9001:storage-minio:9001 deploy@prod
 3. Add Forgejo secrets `FOO_DB_PASSWORD`, `FOO_S3_ACCESS_KEY`, `FOO_S3_SECRET_KEY`
    to this repo *and* to the foo repo.
 4. Extend `init/postgres-init.sh` with `ensure_role "foo" ...` and `ensure_db "foo" "foo"`.
-5. Extend `init/minio-bootstrap.sh` with `ensure_bucket "foo-..."`,
-   `write_policy "foo-...-rw" "foo-..."`, `ensure_user ... "foo-...-rw"`.
+5. Extend `init/garage-bootstrap.sh` with `ensure_bucket "foo-..."`, an
+   `ensure_key foo ...` and a `grant` for it.
 6. Pass the new envs through in `docker-compose.yml` and the deploy workflow.
 7. Push.
 
 ## State and backups
 
-Two named Docker volumes are stateful:
+Three named Docker volumes are stateful:
 
 - `pg_data` (Postgres data dir) — `/var/lib/docker/volumes/storage_pg_data/_data`
-- `minio_data` (S3 object bytes) — `/var/lib/docker/volumes/storage_minio_data/_data`
+- `garage_data` (S3 object bytes) and `garage_meta` (Garage's database,
+  with a snapshot every 6 h) — `/var/lib/docker/volumes/storage_garage_*/_data`
 
-Server-level backup covers both. If you ever move off whole-server backup,
-the minimum is `pg_dumpall` for Postgres and `mc mirror` for MinIO.
+`/opt/storage/secrets/garage_rpc_secret` (Garage's node-to-node and CLI
+secret) is made on the host at the first deploy and is not in Forgejo.
+
+Server-level backup covers all three. If you ever move off whole-server backup,
+the minimum is `pg_dumpall` for Postgres and `rclone sync` of each bucket.
+
+The old `storage_minio_data` volume is still on the host from before the move;
+nothing uses it.
 
 ## Secrets
 
diff --git a/SECRETS.md b/SECRETS.md
index 72cc270..0c4dbe7 100644
--- a/SECRETS.md
+++ b/SECRETS.md
@@ -8,20 +8,20 @@ domain-specific.
 
 ```sh
 openssl rand -hex 32       # 64-char hex, good for any password/key
-openssl rand -base64 24    # 32-char base64, fine for MinIO root password
 ```
 
-For MinIO **access keys** prefer alphanumeric, 20 chars:
+For a Garage **key pair** (an id of `GK` + 24 hex digits, a secret of 64):
 
 ```sh
-LC_ALL=C tr -dc 'A-Z0-9' </dev/urandom | head -c 20; echo
+printf 'GK%s\n' "$(openssl rand -hex 12)"
+openssl rand -hex 32
 ```
 
-For MinIO **secret keys**, 40+ chars:
-
-```sh
-LC_ALL=C tr -dc 'A-Za-z0-9' </dev/urandom | head -c 40; echo
-```
+The pairs in use predate Garage and are not in that format. The bootstrap and
+the apps' deploys hash such a value into it (`garage_key_id`, `garage_secret`
+in `init/garage-bootstrap.sh`), so the key Garage knows differs from the
+secret's text. A value already in Garage's format is used as it is: use the
+snippet above whenever a pair is rotated.
 
 ## Required secrets
 
@@ -41,13 +41,6 @@ Reuse the same key the asxpio repo uses.
 |---------------------|--------------------------------------------------------------------------------|
 | `POSTGRES_PASSWORD` | Postgres superuser password. `openssl rand -hex 32`. Used only by init/admin.  |
 
-### MinIO
-
-| Secret                  | Value                                                                |
-|-------------------------|----------------------------------------------------------------------|
-| `MINIO_ROOT_USER`       | Root username, e.g. `storage-admin`. Any string ≥ 3 chars.           |
-| `MINIO_ROOT_PASSWORD`   | Root password. ≥ 8 chars; use `openssl rand -base64 24`.             |
-
 ### Per-app credentials provisioned for asxpio
 
 These are also added (with the same values) on the **asxpio repo** so the app
@@ -56,37 +49,42 @@ can connect.
 | Secret                  | Value                                                          |
 |-------------------------|----------------------------------------------------------------|
 | `ASXPIO_DB_PASSWORD`    | Postgres password for the `asxpio` role. `openssl rand -hex 32`.|
-| `ASXPIO_S3_ACCESS_KEY`  | 20-char alphanumeric, see snippet above.                       |
-| `ASXPIO_S3_SECRET_KEY`  | 40-char alphanumeric, see snippet above.                       |
+| `ASXPIO_S3_ACCESS_KEY`  | S3 key id, see above.                                          |
+| `ASXPIO_S3_SECRET_KEY`  | S3 secret, see above.                                          |
 
 ### Per-app credentials provisioned for datacenterbingo
 
 These are read-only:
 - Role `bingo` gets SELECT on `bingo_datacenters` and `bingo_incidents` in
   the `asxpio` DB, and `default_transaction_read_only`.
-- The MinIO user may only `GetObject` in the `datacenterbingo-photos` bucket.
-  asxpio's user has read-write on that bucket for the admin's uploads.
+- The S3 key may only read the `datacenterbingo-photos` bucket.
+  asxpio's key has read-write on that bucket for the admin's uploads.
 
 The same values go on the **datacenterbingo repo**.
 
 | Secret                 | Value                                                  |
 |------------------------|--------------------------------------------------------|
 | `BINGO_DB_PASSWORD`    | Postgres password for the `bingo` role. `openssl rand -hex 32`. |
-| `BINGO_S3_ACCESS_KEY`  | 20-char alphanumeric.                                  |
-| `BINGO_S3_SECRET_KEY`  | 40-char alphanumeric.                                  |
+| `BINGO_S3_ACCESS_KEY`  | S3 key id.                                             |
+| `BINGO_S3_SECRET_KEY`  | S3 secret.                                             |
 
 ### Uploader for the `public` bucket
 
 Objects in `public` are readable by anyone at `https://s3.asxp.io/public/<key>`
-(no anonymous listing). This user may only put and delete there. Optional:
-without the secrets the bucket exists and only root can upload.
+(no anonymous listing). This key may only put and delete there. Optional:
+without the secrets the bucket exists and nothing can upload.
 
 | Secret                  | Value                 |
 |-------------------------|-----------------------|
-| `PUBLIC_S3_ACCESS_KEY`  | 20-char alphanumeric. |
-| `PUBLIC_S3_SECRET_KEY`  | 40-char alphanumeric. |
+| `PUBLIC_S3_ACCESS_KEY`  | S3 key id.            |
+| `PUBLIC_S3_SECRET_KEY`  | S3 secret.            |
+
+The local AWS profile needs the pair as Garage knows it. For a pair not in
+Garage's format, that is the hashed form:
 
 ```sh
+printf 'GK%s\n' "$(printf '%s' "$PUBLIC_S3_ACCESS_KEY" | sha256sum | cut -c1-24)"
+printf '%s' "$PUBLIC_S3_SECRET_KEY" | sha256sum | cut -c1-64
 aws --endpoint-url https://s3.asxp.io --profile asxp-public \
   s3 cp shot.png s3://public/shot.png
 ```
@@ -96,13 +94,14 @@ deploy storage again after asxpio's first boot.
 
 ## After adding secrets
 
-1. Push to `main`. The Forgejo workflow brings Postgres and MinIO up, then runs
-   `postgres-init` (creates `asxpio` role + DB) and `minio-init` (creates
-   `asxpio-invoices` bucket and an `asxpio`-scoped user).
+1. Push to `main`. The Forgejo workflow brings Postgres and Garage up, then runs
+   `postgres-init` (creates `asxpio` role + DB) and `init/garage-bootstrap.sh`
+   (creates the buckets and the per-app keys).
 2. Confirm both init containers exited 0:
    ```sh
    ssh deploy@prod 'docker compose -f /opt/storage/docker-compose.yml ps'
-   ssh deploy@prod 'docker logs storage-postgres-init storage-minio-init'
+   ssh deploy@prod 'docker logs storage-postgres-init'
+   ssh deploy@prod 'docker exec storage-garage /garage key list'
    ```
 3. Add the matching `ASXPIO_DB_PASSWORD`, `ASXPIO_S3_ACCESS_KEY`,
    `ASXPIO_S3_SECRET_KEY` (plus `ADMIN_USER`, `ADMIN_PASSWORD`) on the asxpio
@@ -111,9 +110,7 @@ deploy storage again after asxpio's first boot.
 
 ## Rotation
 
-- **`POSTGRES_PASSWORD` / `MINIO_ROOT_PASSWORD`**: update secret, push. Init
-  scripts only touch *per-app* roles, so the root passwords are applied on
-  container restart (Postgres) or via the env on next boot (MinIO). For MinIO
-  root rotation, recreating the container is enough.
+- **`POSTGRES_PASSWORD`**: update secret, push. Init scripts only touch
+  *per-app* roles, so the superuser password is applied on container restart.
 - **Per-app password / keys**: update both repos' secrets, push storage first
   (re-provisions the role/user with the new password), then push the consumer.
diff --git a/docker-compose.yml b/docker-compose.yml
index 05cc453..bba3f50 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -36,29 +36,6 @@ services:
       - storage
     restart: "no"
 
-  # Kept only as the way back until Garage has proven itself: not routed,
-  # not bootstrapped, its data frozen as of the switch.
-  minio:
-    image: minio/minio:latest
-    container_name: storage-minio
-    restart: unless-stopped
-    command: server /data --console-address ":9001"
-    environment:
-      MINIO_ROOT_USER: ${MINIO_ROOT_USER}
-      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
-      MINIO_BROWSER_REDIRECT_URL: https://s3.asxp.io
-    volumes:
-      - minio_data:/data
-    networks:
-      - storage
-    healthcheck:
-      test: ["CMD", "mc", "ready", "local"]
-      interval: 5s
-      timeout: 5s
-      retries: 10
-    labels:
-      - "traefik.enable=false"
-
   garage:
     image: dxflrs/garage:v2.5.0
     container_name: storage-garage
@@ -103,7 +80,6 @@ services:
 
 volumes:
   pg_data:
-  minio_data:
   garage_meta:
   garage_data:
 
diff --git a/flake.nix b/flake.nix
index 1859f9f..d2df8e1 100644
--- a/flake.nix
+++ b/flake.nix
@@ -1,5 +1,5 @@
 {
-  description = "storage — Postgres + MinIO for asxp.io";
+  description = "storage — Postgres + Garage for asxp.io";
 
   inputs = {
     nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
@@ -14,7 +14,8 @@
       devShells.${system}.default = pkgs.mkShell {
         buildInputs = [
           pkgs.postgresql_17
-          pkgs.minio-client
+          pkgs.awscli2
+          pkgs.rclone
           pkgs.docker-compose
           pkgs.openssl
         ];
@@ -29,7 +30,7 @@
           echo "storage dev shell ready"
           echo "  docker compose up -d              - bring stack up"
           echo "  psql -h localhost ...             - if you forward 5432"
-          echo "  mc alias set local ...            - point mc at MinIO"
+          echo "  aws --endpoint-url https://s3.asxp.io s3 ...  - talk to Garage"
         '';
       };
     };
diff --git a/init/minio-bootstrap.sh b/init/minio-bootstrap.sh
deleted file mode 100644
index 893c720..0000000
--- a/init/minio-bootstrap.sh
+++ /dev/null
@@ -1,148 +0,0 @@
-#!/bin/sh
-# Idempotent MinIO bootstrap: bucket + per-app service account scoped to that
-# bucket. Re-running just refreshes the policy and resets the access key.
-set -eu
-
-mc alias set local http://minio:9000 "${MINIO_ROOT_USER}" "${MINIO_ROOT_PASSWORD}"
-
-ensure_bucket() {
-  bucket="$1"
-  if ! mc ls "local/${bucket}" >/dev/null 2>&1; then
-    mc mb "local/${bucket}"
-  fi
-  mc anonymous set none "local/${bucket}" >/dev/null
-}
-
-write_policy() {
-  name="$1"
-  bucket="$2"
-  cat > "/tmp/${name}.json" <<EOF
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Effect": "Allow",
-      "Action": ["s3:ListBucket", "s3:GetBucketLocation"],
-      "Resource": ["arn:aws:s3:::${bucket}"]
-    },
-    {
-      "Effect": "Allow",
-      "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
-      "Resource": ["arn:aws:s3:::${bucket}/*"]
-    }
-  ]
-}
-EOF
-  mc admin policy create local "${name}" "/tmp/${name}.json" 2>/dev/null \
-    || mc admin policy update local "${name}" "/tmp/${name}.json"
-}
-
-# Read-only on one prefix of a bucket.
-write_ro_prefix_policy() {
-  name="$1"
-  bucket="$2"
-  prefix="$3"
-  cat > "/tmp/${name}.json" <<EOF
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Effect": "Allow",
-      "Action": ["s3:GetObject"],
-      "Resource": ["arn:aws:s3:::${bucket}/${prefix}*"]
-    }
-  ]
-}
-EOF
-  mc admin policy create local "${name}" "/tmp/${name}.json" 2>/dev/null \
-    || mc admin policy update local "${name}" "/tmp/${name}.json"
-}
-
-# World-readable by object key; no anonymous listing or writes.
-ensure_public_bucket() {
-  bucket="$1"
-  if ! mc ls "local/${bucket}" >/dev/null 2>&1; then
-    mc mb "local/${bucket}"
-  fi
-  cat > "/tmp/${bucket}-anonymous.json" <<EOF
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Effect": "Allow",
-      "Principal": {"AWS": ["*"]},
-      "Action": ["s3:GetObject"],
-      "Resource": ["arn:aws:s3:::${bucket}/*"]
-    }
-  ]
-}
-EOF
-  mc anonymous set-json "/tmp/${bucket}-anonymous.json" "local/${bucket}" >/dev/null
-}
-
-# Put and delete only: the holder can publish and retract, not read or list.
-write_upload_policy() {
-  name="$1"
-  bucket="$2"
-  cat > "/tmp/${name}.json" <<EOF
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Effect": "Allow",
-      "Action": ["s3:PutObject", "s3:DeleteObject"],
-      "Resource": ["arn:aws:s3:::${bucket}/*"]
-    }
-  ]
-}
-EOF
-  mc admin policy create local "${name}" "/tmp/${name}.json" 2>/dev/null \
-    || mc admin policy update local "${name}" "/tmp/${name}.json"
-}
-
-# $3: one or more policy names, space-separated.
-ensure_user() {
-  access="$1"
-  secret="$2"
-  policies="$3"
-  if mc admin user info local "${access}" >/dev/null 2>&1; then
-    # Rotate secret to match desired state.
-    mc admin user remove local "${access}" >/dev/null
-  fi
-  mc admin user add local "${access}" "${secret}"
-  # shellcheck disable=SC2086 # word-split the policy list on purpose
-  mc admin policy attach local ${policies} --user "${access}" 2>/dev/null || true
-}
-
-ensure_bucket "asxpio-invoices"
-write_policy  "asxpio-invoices-rw" "asxpio-invoices"
-
-# Datacenter bingo photos: uploaded and resized by asxpio's admin, served by
-# datacenterbingo. Keys keep their bingo/ prefix, so DB rows stay valid.
-ensure_bucket "datacenterbingo-photos"
-write_policy  "datacenterbingo-photos-rw" "datacenterbingo-photos"
-write_ro_prefix_policy "datacenterbingo-photos-ro" "datacenterbingo-photos" ""
-
-ensure_user   "${ASXPIO_S3_ACCESS_KEY}" "${ASXPIO_S3_SECRET_KEY}" "asxpio-invoices-rw datacenterbingo-photos-rw"
-
-if [ -n "${BINGO_S3_ACCESS_KEY:-}" ] && [ -n "${BINGO_S3_SECRET_KEY:-}" ]; then
-  ensure_user   "${BINGO_S3_ACCESS_KEY}" "${BINGO_S3_SECRET_KEY}" "datacenterbingo-photos-ro"
-else
-  echo "minio-init: BINGO_S3_* unset, skipping bingo user"
-fi
-
-# Files shared by link: https://s3.asxp.io/public/<key>. Everything in this
-# bucket is readable by anyone who knows the key.
-ensure_public_bucket "public"
-write_upload_policy  "public-upload" "public"
-
-if [ -n "${PUBLIC_S3_ACCESS_KEY:-}" ] && [ -n "${PUBLIC_S3_SECRET_KEY:-}" ]; then
-  ensure_user   "${PUBLIC_S3_ACCESS_KEY}" "${PUBLIC_S3_SECRET_KEY}" "public-upload"
-else
-  echo "minio-init: PUBLIC_S3_* unset, skipping public uploader"
-fi
-
-# Leftover from before the photos had their own bucket.
-mc admin policy remove local "asxpio-invoices-bingo-ro" >/dev/null 2>&1 || true
-
-echo "minio-init: done"