From 3f279236c9f5b87b59c947f6730078be256428d7 Mon Sep 17 00:00:00 2001 From: Kirill Shklyaev Date: Wed, 16 Sep 2026 21:28:02 +0300 Subject: [PATCH] feat!: replace MinIO with Garage as the S3 backend MinIO declared its open-source edition unmaintained and now points users to its paid product, so any-sync-filenode blob storage moves to Garage v2.3, which is maintained, single-binary and built for self-hosting. Garage runs with --single-node --default-bucket --default-access-key, so it creates its bucket and access key on first start and the mc-based create-bucket service is gone. Its config is static (garage/garage.toml) and the RPC secret comes from the environment, which keeps garage independent of any-sync-init and therefore available while the migration guard blocks the rest of the stack. Existing installs are protected by two barriers and a non-destructive migration: - an outdated .env fails `docker compose` immediately via required-variable syntax (GARAGE_VERSION, GARAGE_RPC_SECRET, S3_BUCKET) - the new s3-guard service refuses to start the stack while objects remain in storage/minio without a completed-migration marker; any-sync-init depends on it, so nothing else starts either - `docker compose --profile migrate run --rm s3-migrate` (or `make migrate`) copies every object with rclone, verifies checksums and only then writes the marker; MinIO data is kept as a backup and removed by an explicit `make cleanLegacyMinio` The MINIO_* variables are renamed to S3_* (S3_ENDPOINT, S3_BUCKET, S3_REGION, S3_LEGACY_BUCKET), which is what makes the first barrier possible. The MinIO service is retained under the "migrate" profile only. Migration instructions live in the wiki Upgrade Guide. BREAKING CHANGE: the MINIO_* variables in .env are replaced by S3_*/GARAGE_* and existing installations must run the MinIO -> Garage data migration before the stack will start. See the Upgrade Guide for the steps. Closes #178 --- .env.example | 38 ++++++++---- Makefile | 11 ++++ README.md | 12 +++- docker-compose.yml | 83 ++++++++++++++++++++------ docker-generateconfig/etc/filenode.yml | 8 +-- docker-generateconfig/etc/node-1.yml | 6 +- docker-generateconfig/etc/node-2.yml | 6 +- docker-generateconfig/etc/node-3.yml | 6 +- docker-generateconfig/migrate-s3.sh | 79 ++++++++++++++++++++++++ docker-generateconfig/s3-guard.sh | 60 +++++++++++++++++++ garage/garage.toml | 19 ++++++ tests/main.sh | 8 +-- tests/run.d/setMinioPort.sh | 5 -- tests/variables | 2 - 14 files changed, 289 insertions(+), 54 deletions(-) create mode 100755 docker-generateconfig/migrate-s3.sh create mode 100755 docker-generateconfig/s3-guard.sh create mode 100644 garage/garage.toml delete mode 100644 tests/run.d/setMinioPort.sh diff --git a/.env.example b/.env.example index 0d1c20c..1f9bf41 100644 --- a/.env.example +++ b/.env.example @@ -16,7 +16,12 @@ ANYTYPE_CLI_VERSION=latest MONGO_VERSION=7.0.28 REDIS_VERSION=7.2.0-v6 -MINIO_VERSION=RELEASE.2025-09-07T16-13-09Z.hotfix.7aa24e772 +GARAGE_VERSION=v2.3.0 + +# Used only by the one-off "migrate" profile (see the Upgrade Guide wiki page). +# MinIO is no longer part of the running stack. +MINIO_LEGACY_VERSION=RELEASE.2025-09-07T16-13-09Z.hotfix.7aa24e772 +RCLONE_VERSION=1.71.1 # ----------------------------------------------------------------------------- @@ -78,7 +83,7 @@ ANY_SYNC_COORDINATOR_DEFAULT_LIMITS_SHARED_SPACES_LIMIT=1000 # ----------------------------------------------------------------------------- # any-sync-filenode -# Handles binary file (blob) storage, backed by MinIO/S3. +# Handles binary file (blob) storage, backed by S3 (Garage). # ----------------------------------------------------------------------------- ANY_SYNC_FILENODE_HOST=any-sync-filenode ANY_SYNC_FILENODE_PORT=1005 @@ -133,15 +138,28 @@ REDIS_MAXMEMORY=256mb # ----------------------------------------------------------------------------- -# MinIO (S3-compatible object storage) +# S3-compatible object storage (Garage) # Stores files managed by any-sync-filenode. -# ----------------------------------------------------------------------------- -AWS_ACCESS_KEY_ID=minio_access_key -AWS_SECRET_ACCESS_KEY=minio_secret_key -MINIO_BUCKET=minio-bucket - -MINIO_PORT=9000 -MINIO_WEB_PORT=9001 +# Replaces MinIO — see the Upgrade Guide wiki page before upgrading an existing install. +# ----------------------------------------------------------------------------- +AWS_ACCESS_KEY_ID=any_sync_access_key +AWS_SECRET_ACCESS_KEY=any_sync_secret_key + +# Internal address of the S3 service. Point it to an external S3-compatible +# storage if you do not want to use the bundled Garage container. +# The port Garage listens on is set in garage/garage.toml. +S3_ENDPOINT=http://garage:3900 +S3_BUCKET=any-sync-files +S3_REGION=us-east-1 + +# Garage cluster RPC secret (64 hex chars). Port 3901 is not published to the host, +# so the default is safe for a single-node setup. Change it if you expose that port: +# openssl rand -hex 32 +GARAGE_RPC_SECRET=6b6579206f6e6c792075736564206c6f63616c6c7920627920612073696e676c + +# Name of the old MinIO bucket, used by the one-off migration from MinIO. +# Only change this if you customized MINIO_BUCKET before the migration. +S3_LEGACY_BUCKET=minio-bucket # ----------------------------------------------------------------------------- diff --git a/Makefile b/Makefile index 129154d..beb017e 100644 --- a/Makefile +++ b/Makefile @@ -41,8 +41,19 @@ restart: down start update: pull down start upgrade: down clean start +# one-off migration of filenode blobs from MinIO to Garage, see the Upgrade Guide wiki page +migrate: + $(DOCKER_COMPOSE) --profile migrate run --rm s3-migrate + $(DOCKER_COMPOSE) --profile migrate rm -sf minio-legacy + STORAGE_DIR := $(shell grep -m1 '^STORAGE_DIR=' .env 2>/dev/null | cut -d= -f2- | tr -d '"') STORAGE_DIR := $(if $(STORAGE_DIR),$(STORAGE_DIR),./storage) cleanEtcStorage: rm -rf etc/ $(STORAGE_DIR) + +# remove the old MinIO data kept as a backup after the migration to Garage +cleanLegacyMinio: + @[ -f $(STORAGE_DIR)/garage/.migrated-from-minio ] || { echo "Error: migration to Garage was not completed — refusing to delete $(STORAGE_DIR)/minio. See https://github.com/anyproto/any-sync-dockercompose/wiki/Upgrade-Guide"; exit 1; } + @echo "⚠️ Removing $(STORAGE_DIR)/minio — make sure your files are accessible in the Anytype client first." + rm -rf $(STORAGE_DIR)/minio diff --git a/README.md b/README.md index a40cf69..7aecb70 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,12 @@ A self-hosted any-sync network, designed for personal use, review, or testing pu > [!WARNING] > Before upgrading, read the [Upgrade Guide](../../wiki/Upgrade-Guide). +> [!IMPORTANT] +> MinIO has been replaced by [Garage](https://garagehq.deuxfleurs.fr/) (see +> [#178](https://github.com/anyproto/any-sync-dockercompose/issues/178)). +> Existing installs must run a one-off data migration — see the [Upgrade Guide](../../wiki/Upgrade-Guide). +> The stack refuses to start until the migration is done; no data is deleted. + ## Requirements - [Docker](https://docs.docker.com/compose/install/) with Compose plugin v2+ @@ -22,12 +28,12 @@ The stack runs the following services: |---|---|---| | `any-sync-coordinator` | Network coordinator, manages spaces and members | MongoDB | | `any-sync-node` ×3 | Document sync nodes | coordinator | -| `any-sync-filenode` | File storage node | coordinator, MinIO, Redis | +| `any-sync-filenode` | File storage node | coordinator, Garage, Redis | | `any-sync-consensusnode` | Consensus for conflict resolution | coordinator | | `netcheck` | Periodic connectivity health monitor | all nodes | | MongoDB | Coordinator state | — | | Redis | Filenode index | — | -| MinIO | S3-compatible object storage for files | — | +| Garage | S3-compatible object storage for files | — | Optional service: `anytype-cli` (commented out in `docker-compose.yml`) — HTTP/gRPC API server for automation. @@ -95,6 +101,8 @@ make upgrade # ⚠️ Full reset: removes containers and volumes, then start make down # Stop and remove containers (data preserved) make clean # ⚠️ docker system prune --all --volumes (removes all Docker data) make cleanEtcStorage # Remove generated ./etc/ configs and ./storage/ +make migrate # One-off migration of file storage from MinIO to Garage (see Upgrade Guide) +make cleanLegacyMinio # Remove the old MinIO data kept as a backup after the migration ``` ## Documentation diff --git a/docker-compose.yml b/docker-compose.yml index 6867662..af88c50 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,9 +12,25 @@ x-anysync-node: &anysync-node condition: service_started services: + # refuse to start the stack while data from a previous MinIO install + # has not been migrated to Garage yet (see the Upgrade Guide wiki page) + s3-guard: + restart: "no" + build: + context: . + dockerfile: Dockerfile-any-sync-init + entrypoint: ["bash", "/code/docker-generateconfig/s3-guard.sh"] + volumes: + - ./.env:/code/.env:ro,Z + - ./docker-generateconfig:/code/docker-generateconfig:ro,Z + - "${STORAGE_DIR}:/code/storage:Z" + # generate and process any-sync configs any-sync-init: restart: "no" + depends_on: + s3-guard: + condition: service_completed_successfully build: context: . dockerfile: Dockerfile-any-sync-init @@ -57,36 +73,67 @@ services: ports: - "127.0.0.1:${REDIS_PORT}:${REDIS_PORT}" - minio: - image: "quay.io/minio/minio:${MINIO_VERSION}" + # S3-compatible object storage for any-sync-filenode. + # The bucket and the access key are created automatically on first start. + garage: + image: "dxflrs/garage:${GARAGE_VERSION:?GARAGE_VERSION is not set - your .env is outdated, see https://github.com/anyproto/any-sync-dockercompose/wiki/Upgrade-Guide}" restart: unless-stopped - command: ["server", "/data", "--console-address", ":${MINIO_WEB_PORT}", "--address", ":${MINIO_PORT}"] + command: ["/garage", "server", "--single-node", "--default-bucket", "--default-access-key"] + environment: + GARAGE_RPC_SECRET: "${GARAGE_RPC_SECRET:?GARAGE_RPC_SECRET is not set - your .env is outdated, see https://github.com/anyproto/any-sync-dockercompose/wiki/Upgrade-Guide}" + GARAGE_DEFAULT_ACCESS_KEY: "${AWS_ACCESS_KEY_ID}" + GARAGE_DEFAULT_SECRET_KEY: "${AWS_SECRET_ACCESS_KEY}" + GARAGE_DEFAULT_BUCKET: "${S3_BUCKET:?S3_BUCKET is not set - your .env is outdated, see https://github.com/anyproto/any-sync-dockercompose/wiki/Upgrade-Guide}" + volumes: + - ./garage/garage.toml:/etc/garage.toml:ro,Z + - "${STORAGE_DIR}/garage:/var/lib/garage:Z" + healthcheck: + test: ["CMD", "/garage", "status"] + interval: 5s + timeout: 10s + retries: 5 + start_period: 10s + + # --- one-off migration from MinIO to Garage, see the Upgrade Guide ----------- + # Not started by "docker compose up"; only by: + # docker compose --profile migrate run --rm s3-migrate + + minio-legacy: + profiles: ["migrate"] + image: "quay.io/minio/minio:${MINIO_LEGACY_VERSION}" + restart: "no" + command: ["server", "/data", "--address", ":9000"] environment: MINIO_ROOT_USER: "${AWS_ACCESS_KEY_ID}" MINIO_ROOT_PASSWORD: "${AWS_SECRET_ACCESS_KEY}" volumes: - "${STORAGE_DIR}/minio:/data:Z" healthcheck: - test: bash -c ':> /dev/tcp/127.0.0.1/${MINIO_PORT}' || exit 1 + test: bash -c ':> /dev/tcp/127.0.0.1/9000' || exit 1 interval: 5s timeout: 10s retries: 3 - networks: - default: - aliases: - - "${MINIO_BUCKET}.minio" # . - create-bucket: + s3-migrate: + profiles: ["migrate"] + image: "rclone/rclone:${RCLONE_VERSION}" restart: "no" - image: quay.io/minio/mc:latest depends_on: - - minio - entrypoint: > - /bin/sh -c " - until (/usr/bin/mc alias set minio http://minio:${MINIO_PORT} '${AWS_ACCESS_KEY_ID}' '${AWS_SECRET_ACCESS_KEY}') do echo '...waiting...' && sleep 1; done; - /usr/bin/mc mb minio/${MINIO_BUCKET}; - exit 0; - " + minio-legacy: + condition: service_healthy + garage: + condition: service_healthy + environment: + AWS_ACCESS_KEY_ID: "${AWS_ACCESS_KEY_ID}" + AWS_SECRET_ACCESS_KEY: "${AWS_SECRET_ACCESS_KEY}" + S3_BUCKET: "${S3_BUCKET}" + S3_REGION: "${S3_REGION}" + S3_LEGACY_BUCKET: "${S3_LEGACY_BUCKET}" + entrypoint: ["/bin/sh", "/code/docker-generateconfig/migrate-s3.sh"] + volumes: + - ./docker-generateconfig:/code/docker-generateconfig:ro,Z + - "${STORAGE_DIR}:/code/storage:Z" + # --------------------------------------------------------------------------- any-sync-coordinator_bootstrap: image: "ghcr.io/anyproto/any-sync-coordinator:${ANY_SYNC_COORDINATOR_VERSION}" @@ -124,7 +171,7 @@ services: depends_on: redis: condition: service_healthy - minio: + garage: condition: service_healthy any-sync-coordinator: condition: service_started diff --git a/docker-generateconfig/etc/filenode.yml b/docker-generateconfig/etc/filenode.yml index 21fde1c..22a1f86 100644 --- a/docker-generateconfig/etc/filenode.yml +++ b/docker-generateconfig/etc/filenode.yml @@ -1,10 +1,10 @@ s3Store: - bucket: %MINIO_BUCKET% - indexBucket: %MINIO_BUCKET% + bucket: %S3_BUCKET% + indexBucket: %S3_BUCKET% maxThreads: 16 profile: default - region: us-east-1 - endpoint: http://minio:%MINIO_PORT% + region: %S3_REGION% + endpoint: %S3_ENDPOINT% forcePathStyle: true # 'true' for self-hosted S3 Object Storage redis: diff --git a/docker-generateconfig/etc/node-1.yml b/docker-generateconfig/etc/node-1.yml index 54ea158..b40bdae 100644 --- a/docker-generateconfig/etc/node-1.yml +++ b/docker-generateconfig/etc/node-1.yml @@ -31,9 +31,9 @@ nodeSync: s3Store: enabled: false - region: us-east-1 - endpoint: http://minio:%MINIO_PORT% - bucket: %MINIO_BUCKET% + region: %S3_REGION% + endpoint: %S3_ENDPOINT% + bucket: %S3_BUCKET% keyPrefix: "node_storage/node-1" forcePathStyle: true # 'true' for self-hosted S3 Object Storage diff --git a/docker-generateconfig/etc/node-2.yml b/docker-generateconfig/etc/node-2.yml index e147fc6..ee3dd1b 100644 --- a/docker-generateconfig/etc/node-2.yml +++ b/docker-generateconfig/etc/node-2.yml @@ -31,9 +31,9 @@ nodeSync: s3Store: enabled: false - region: us-east-1 - endpoint: http://minio:%MINIO_PORT% - bucket: %MINIO_BUCKET% + region: %S3_REGION% + endpoint: %S3_ENDPOINT% + bucket: %S3_BUCKET% keyPrefix: "node_storage/node-2" forcePathStyle: true # 'true' for self-hosted S3 Object Storage diff --git a/docker-generateconfig/etc/node-3.yml b/docker-generateconfig/etc/node-3.yml index 1eceb46..f6f2a23 100644 --- a/docker-generateconfig/etc/node-3.yml +++ b/docker-generateconfig/etc/node-3.yml @@ -31,9 +31,9 @@ nodeSync: s3Store: enabled: false - region: us-east-1 - endpoint: http://minio:%MINIO_PORT% - bucket: %MINIO_BUCKET% + region: %S3_REGION% + endpoint: %S3_ENDPOINT% + bucket: %S3_BUCKET% keyPrefix: "node_storage/node-3" forcePathStyle: true # 'true' for self-hosted S3 Object Storage diff --git a/docker-generateconfig/migrate-s3.sh b/docker-generateconfig/migrate-s3.sh new file mode 100755 index 0000000..4429a04 --- /dev/null +++ b/docker-generateconfig/migrate-s3.sh @@ -0,0 +1,79 @@ +#!/bin/sh +# One-off migration of any-sync-filenode blobs from MinIO to Garage. +# Runs inside the rclone image, started by: +# docker compose --profile migrate run --rm s3-migrate +set -e + +SRC_BUCKET="${S3_LEGACY_BUCKET:-minio-bucket}" +DST_BUCKET="${S3_BUCKET}" +MARKER="/code/storage/garage/.migrated-from-minio" + +# rclone remotes are configured entirely through the environment +export RCLONE_CONFIG_SRC_TYPE=s3 +export RCLONE_CONFIG_SRC_PROVIDER=Minio +export RCLONE_CONFIG_SRC_ENDPOINT="http://minio-legacy:9000" +export RCLONE_CONFIG_SRC_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}" +export RCLONE_CONFIG_SRC_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}" +export RCLONE_CONFIG_SRC_REGION="${S3_REGION:-us-east-1}" +export RCLONE_CONFIG_SRC_FORCE_PATH_STYLE=true + +export RCLONE_CONFIG_DST_TYPE=s3 +export RCLONE_CONFIG_DST_PROVIDER=Other +export RCLONE_CONFIG_DST_ENDPOINT="http://garage:3900" +export RCLONE_CONFIG_DST_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}" +export RCLONE_CONFIG_DST_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}" +export RCLONE_CONFIG_DST_REGION="${S3_REGION:-us-east-1}" +export RCLONE_CONFIG_DST_FORCE_PATH_STYLE=true + +if [ -f "${MARKER}" ]; then + echo "INFO: migration already completed, nothing to do:" + sed 's/^/INFO: /' "${MARKER}" + exit 0 +fi + +if [ ! -d /code/storage/minio ]; then + echo "INFO: no legacy MinIO data found, nothing to migrate" + exit 0 +fi + +echo "INFO: copying objects src:${SRC_BUCKET} -> dst:${DST_BUCKET}" +rclone copy "src:${SRC_BUCKET}" "dst:${DST_BUCKET}" \ + --checksum --transfers 8 --checkers 8 --stats 5s --stats-one-line + +echo "INFO: verifying (checksum comparison, one-way)" +rclone check "src:${SRC_BUCKET}" "dst:${DST_BUCKET}" --one-way --checksum + +SRC_COUNT=$(rclone size "src:${SRC_BUCKET}" --json | sed 's/.*"count":\([0-9]*\).*/\1/') +DST_COUNT=$(rclone size "dst:${DST_BUCKET}" --json | sed 's/.*"count":\([0-9]*\).*/\1/') +SRC_BYTES=$(rclone size "src:${SRC_BUCKET}" --json | sed 's/.*"bytes":\([0-9]*\).*/\1/') + +if [ "${SRC_COUNT}" -gt "${DST_COUNT}" ]; then + echo "ERROR: object count mismatch: src=${SRC_COUNT} dst=${DST_COUNT}, marker not written" >&2 + exit 1 +fi + +mkdir -p /code/storage/garage +cat > "${MARKER}" < Garage migration already done:" + sed 's/^/INFO: /' "${MARKER}" + exit 0 +fi + +OBJECT_COUNT=$(find "${LEGACY_BUCKET_DIR}" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | wc -l) + +if [[ "${OBJECT_COUNT}" -eq 0 ]]; then + echo "INFO: legacy MinIO directory '${LEGACY_DIR}' is empty, nothing to migrate" + install -d ./storage/garage + echo "nothing to migrate: legacy MinIO bucket was empty ($(date -u +%Y-%m-%dT%H:%M:%SZ))" > "${MARKER}" + exit 0 +fi + +cat >&2 </dev/null | grep -q "${S3_BUCKET}"; then + echo -e "${GREEN} Garage bucket creation - OK [✔] ${NC}" else - echo -e "${RED} Minio bucket creation - FAILED [✖] ${NC}" + echo -e "${RED} Garage bucket creation - FAILED [✖] ${NC}" restoreBackup exit 1 fi diff --git a/tests/run.d/setMinioPort.sh b/tests/run.d/setMinioPort.sh deleted file mode 100644 index 007a13c..0000000 --- a/tests/run.d/setMinioPort.sh +++ /dev/null @@ -1,5 +0,0 @@ -# Writes Minio port configurations to .env.override -cat < $PROJECT_DIR/.env.override -MINIO_PORT=$MINIO_PORT -MINIO_WEB_PORT=$MINIO_WEB_PORT -EOF diff --git a/tests/variables b/tests/variables index e72ec81..415cfea 100644 --- a/tests/variables +++ b/tests/variables @@ -12,5 +12,3 @@ ANY_SYNC_FILENODE_VERSION=("v0.8.1" "prod" "stage1") ANY_SYNC_COORDINATOR_VERSION=("v0.4.2" "prod" "stage1") ANY_SYNC_CONSENSUSNODE_VERSION=("v0.2.0" "prod" "stage1") -MINIO_PORT=2000 -MINIO_WEB_PORT=3000