From dc04ff3cce68dce68720c0551aaf2c96e406133d Mon Sep 17 00:00:00 2001 From: Sergei Poljanski Date: Wed, 11 Nov 2026 03:41:03 +0400 Subject: [PATCH] Add Garage beside MinIO MinIO is abandoned upstream and its images are gone from the registries, so the object store cannot be rebuilt or updated. Garage takes its place. This is the first step: Garage runs as one node next to MinIO, with the same three buckets and one key per app, and is not routed yet. MinIO keeps serving s3.asxp.io and is no longer bootstrapped, so its users stay as they are until the data is copied and the apps are switched. Garage only accepts keys in its own format. The bootstrap hashes the existing secrets into that format, as the apps' deploys will, so no secret has to change for the move; a secret already in Garage's format is used as it is. Garage has no anonymous S3 access. The public bucket is marked as a website instead, to be served through Garage's web endpoint. Assisted-by: Claude:fable-5-2 --- .forgejo/workflows/deploy.yaml | 21 +++++-- docker-compose.yml | 40 +++++++------ garage.toml | 30 ++++++++++ init/garage-bootstrap.sh | 103 +++++++++++++++++++++++++++++++++ 4 files changed, 171 insertions(+), 23 deletions(-) create mode 100644 garage.toml create mode 100644 init/garage-bootstrap.sh diff --git a/.forgejo/workflows/deploy.yaml b/.forgejo/workflows/deploy.yaml index 4491784..ace0d22 100644 --- a/.forgejo/workflows/deploy.yaml +++ b/.forgejo/workflows/deploy.yaml @@ -23,7 +23,7 @@ jobs: REMOTE_HOST: ${{ secrets.DEPLOY_IP }} REMOTE_USER: ${{ secrets.DEPLOY_USER }} ARGS: "-rltgoDzvO --delete" - SOURCE: "docker-compose.yml init" + SOURCE: "docker-compose.yml garage.toml init" TARGET: "/opt/storage/" - name: Write .env from secrets and bring stack up @@ -63,15 +63,26 @@ jobs: PUBLIC_S3_SECRET_KEY=${PUBLIC_S3_SECRET_KEY} EOF chmod 600 /opt/storage/.env - chmod +x /opt/storage/init/postgres-init.sh /opt/storage/init/minio-bootstrap.sh + chmod +x /opt/storage/init/postgres-init.sh /opt/storage/init/garage-bootstrap.sh + + # Garage's RPC secret never leaves the host: made once, kept + # outside the rsynced paths. + mkdir -p /opt/storage/secrets + if [ ! -s /opt/storage/secrets/garage_rpc_secret ]; then + head -c 32 /dev/urandom | od -An -tx1 | tr -d ' \n' > /opt/storage/secrets/garage_rpc_secret + fi + chmod 600 /opt/storage/secrets/garage_rpc_secret cd /opt/storage # minio/minio and minio/mc are gone from Docker Hub (upstream # abandoned); the host's cached copies are all there is. docker compose pull --ignore-pull-failures - docker compose up -d postgres minio - # Run init containers (idempotent) and wait for them to exit. + docker compose up -d postgres minio garage + # Run the init steps (idempotent). docker compose up --exit-code-from postgres-init postgres-init - docker compose up --exit-code-from minio-init minio-init + ASXPIO_S3_ACCESS_KEY="${ASXPIO_S3_ACCESS_KEY}" ASXPIO_S3_SECRET_KEY="${ASXPIO_S3_SECRET_KEY}" \ + BINGO_S3_ACCESS_KEY="${BINGO_S3_ACCESS_KEY}" BINGO_S3_SECRET_KEY="${BINGO_S3_SECRET_KEY}" \ + PUBLIC_S3_ACCESS_KEY="${PUBLIC_S3_ACCESS_KEY}" PUBLIC_S3_SECRET_KEY="${PUBLIC_S3_SECRET_KEY}" \ + sh init/garage-bootstrap.sh echo "Storage stack deployed." diff --git a/docker-compose.yml b/docker-compose.yml index 6aed8a7..ae53edb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -65,31 +65,35 @@ services: - "traefik.http.routers.minio-s3.service=minio-s3" - "traefik.http.services.minio-s3.loadbalancer.server.port=9000" - minio-init: - image: minio/mc:latest - container_name: storage-minio-init - depends_on: - minio: - condition: service_healthy - environment: - MINIO_ROOT_USER: ${MINIO_ROOT_USER} - MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} - ASXPIO_S3_ACCESS_KEY: ${ASXPIO_S3_ACCESS_KEY} - ASXPIO_S3_SECRET_KEY: ${ASXPIO_S3_SECRET_KEY} - BINGO_S3_ACCESS_KEY: ${BINGO_S3_ACCESS_KEY} - BINGO_S3_SECRET_KEY: ${BINGO_S3_SECRET_KEY} - PUBLIC_S3_ACCESS_KEY: ${PUBLIC_S3_ACCESS_KEY} - PUBLIC_S3_SECRET_KEY: ${PUBLIC_S3_SECRET_KEY} + # Replaces MinIO (abandoned upstream; its images are gone from the + # registries). Not routed yet: MinIO still answers s3.asxp.io until the data + # is copied and the apps are switched. + garage: + image: dxflrs/garage:v2.5.0 + container_name: storage-garage + restart: unless-stopped + command: ["/garage", "server", "--single-node"] volumes: - - ./init/minio-bootstrap.sh:/bootstrap.sh:ro - entrypoint: ["/bin/sh", "/bootstrap.sh"] + - ./garage.toml:/etc/garage.toml:ro + - ./secrets/garage_rpc_secret:/run/secrets/garage_rpc_secret:ro + - garage_meta:/var/lib/garage/meta + - garage_data:/var/lib/garage/data networks: - storage - restart: "no" + - traefik + healthcheck: + test: ["CMD", "/garage", "status"] + interval: 10s + timeout: 5s + retries: 10 + labels: + - "traefik.enable=false" volumes: pg_data: minio_data: + garage_meta: + garage_data: networks: storage: diff --git a/garage.toml b/garage.toml new file mode 100644 index 0000000..52f9e98 --- /dev/null +++ b/garage.toml @@ -0,0 +1,30 @@ +# Garage, single node. No secrets here: the RPC secret is a file made on the +# host at first deploy (see the deploy workflow). +metadata_dir = "/var/lib/garage/meta" +data_dir = "/var/lib/garage/data" + +# One node and no replica to repair from, so prefer the engine that survives +# an unclean stop, sync writes, and keep metadata snapshots. +db_engine = "sqlite" +metadata_fsync = true +data_fsync = true +metadata_auto_snapshot_interval = "6h" + +replication_factor = 1 + +rpc_bind_addr = "[::]:3901" +rpc_public_addr = "127.0.0.1:3901" +rpc_secret_file = "/run/secrets/garage_rpc_secret" + +[s3_api] +# The apps sign for us-east-1, as they did against MinIO. +s3_region = "us-east-1" +api_bind_addr = "[::]:3900" + +# Anonymous reads. Garage has no anonymous S3 access; a bucket marked as a +# website is served here, picked by Host: .web.garage. Traefik sends +# https://s3.asxp.io/public/ this way. +[s3_web] +bind_addr = "[::]:3902" +root_domain = ".web.garage" +index = "index.html" diff --git a/init/garage-bootstrap.sh b/init/garage-bootstrap.sh new file mode 100644 index 0000000..3963846 --- /dev/null +++ b/init/garage-bootstrap.sh @@ -0,0 +1,103 @@ +#!/bin/sh +# Idempotent Garage bootstrap, run on the host after `docker compose up`: +# buckets, per-app keys and what each key may do. The Garage image has no +# shell, so this drives its CLI through `docker exec`. +# +# Env: ASXPIO_S3_ACCESS_KEY/_SECRET_KEY (required), BINGO_S3_* and +# PUBLIC_S3_* (optional). GARAGE_CONTAINER overrides the container name. +set -eu + +c="${GARAGE_CONTAINER:-storage-garage}" +g() { docker exec "$c" /garage "$@"; } + +# Garage only takes its own key format: an id of GK + 24 hex digits and a +# secret of 64 hex digits. A value already in that format is used as it is; +# anything else (the MinIO-era secrets) is hashed into it, the same way the +# apps' deploy workflows do, so the secrets did not have to change with the +# move. Replace a secret with a native Garage pair whenever it is rotated. +garage_key_id() { + if printf '%s' "$1" | grep -Eq '^GK[0-9a-f]{24}$'; then + printf '%s' "$1" + else + printf 'GK%s' "$(printf '%s' "$1" | sha256sum | cut -c1-24)" + fi +} +garage_secret() { + if printf '%s' "$1" | grep -Eq '^[0-9a-f]{64}$'; then + printf '%s' "$1" + else + printf '%s' "$1" | sha256sum | cut -c1-64 + fi +} + +i=0 +until g status >/dev/null 2>&1; do + i=$((i + 1)) + if [ "$i" -gt 60 ]; then + echo "garage-init: garage did not come up" >&2 + exit 1 + fi + sleep 1 +done + +ensure_bucket() { + g bucket info "$1" >/dev/null 2>&1 || g bucket create "$1" >/dev/null +} + +# $1 name, $2 access key, $3 secret key. Prints the Garage key id. +ensure_key() { + id="$(garage_key_id "$2")" + secret="$(garage_secret "$3")" + if g key info --show-secret "$id" >/dev/null 2>&1; then + if ! g key info --show-secret "$id" | grep -q "$secret"; then + # Rotated secret under the same id. + g key delete --yes "$id" >/dev/null + g key import --yes -n "$1" "$id" "$secret" >/dev/null + fi + else + g key import --yes -n "$1" "$id" "$secret" >/dev/null + fi + printf '%s' "$id" +} + +# $1 bucket, $2 key id, $3 "rw", "ro" or "wo". Sets exactly that. +grant() { + case "$3" in + rw) g bucket allow --read --write "$1" --key "$2" >/dev/null + g bucket deny --owner "$1" --key "$2" >/dev/null ;; + ro) g bucket allow --read "$1" --key "$2" >/dev/null + g bucket deny --write --owner "$1" --key "$2" >/dev/null ;; + wo) g bucket allow --write "$1" --key "$2" >/dev/null + g bucket deny --read --owner "$1" --key "$2" >/dev/null ;; + esac +} + +ensure_bucket "asxpio-invoices" +# Datacenter bingo photos: uploaded and resized by asxpio's admin, served by +# datacenterbingo. +ensure_bucket "datacenterbingo-photos" +# Files shared by link: https://s3.asxp.io/public/. Everything in this +# bucket is readable by anyone who knows the key; nobody can list it. +ensure_bucket "public" +g bucket website --allow public >/dev/null + +asxpio="$(ensure_key asxpio "${ASXPIO_S3_ACCESS_KEY}" "${ASXPIO_S3_SECRET_KEY}")" +grant "asxpio-invoices" "$asxpio" rw +grant "datacenterbingo-photos" "$asxpio" rw + +if [ -n "${BINGO_S3_ACCESS_KEY:-}" ] && [ -n "${BINGO_S3_SECRET_KEY:-}" ]; then + bingo="$(ensure_key bingo "${BINGO_S3_ACCESS_KEY}" "${BINGO_S3_SECRET_KEY}")" + grant "datacenterbingo-photos" "$bingo" ro +else + echo "garage-init: BINGO_S3_* unset, skipping bingo key" +fi + +# Put and delete only: the holder can publish and retract, not read or list. +if [ -n "${PUBLIC_S3_ACCESS_KEY:-}" ] && [ -n "${PUBLIC_S3_SECRET_KEY:-}" ]; then + uploader="$(ensure_key public-upload "${PUBLIC_S3_ACCESS_KEY}" "${PUBLIC_S3_SECRET_KEY}")" + grant "public" "$uploader" wo +else + echo "garage-init: PUBLIC_S3_* unset, skipping public uploader" +fi + +echo "garage-init: done"