From 3385923ccf3959cecf417885c2583cb606a2a4df Mon Sep 17 00:00:00 2001 From: Behrouz Mirabdi Date: Mon, 27 Jul 2026 15:43:40 +0200 Subject: [PATCH 1/5] feat(compose): split CPU base from GPU and CI compose overlays Remove gpus: all and NVIDIA_VISIBLE_DEVICES from the portable base stack so ProSeqGo boots on CPU-only hosts (GitHub Actions, CPU cloud VMs). Add docker-compose.gpu.yml to opt into NVIDIA for embedding-worker, go-prediction-api, and trainer-worker. Add docker-compose.ci.yml to force CAFA_DEVICE=cpu, use CPU PyTorch wheels, and skip postgres-backup/ backup-offload during serving smoke via the ci-skip profile. --- docker-compose.ci.yml | 32 ++++++++++++++++++++++++++++++++ docker-compose.gpu.yml | 19 +++++++++++++++++++ docker-compose.yml | 21 ++++++++++++--------- 3 files changed, 63 insertions(+), 9 deletions(-) create mode 100644 docker-compose.ci.yml create mode 100644 docker-compose.gpu.yml diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml new file mode 100644 index 0000000..ab854a7 --- /dev/null +++ b/docker-compose.ci.yml @@ -0,0 +1,32 @@ +# CI / CPU smoke overlay. Merge with docker-compose.yml: +# docker compose -f docker-compose.yml -f docker-compose.ci.yml up -d --build +# Or use: make ci-up + +services: + embedding-api: + environment: + CAFA_DEVICE: cpu + build: + args: + TORCH_INDEX_URL: https://download.pytorch.org/whl/cpu + + embedding-worker: + environment: + CAFA_DEVICE: cpu + build: + args: + TORCH_INDEX_URL: https://download.pytorch.org/whl/cpu + + go-prediction-api: + environment: + CAFA_DEVICE: cpu + build: + args: + TORCH_INDEX_URL: https://download.pytorch.org/whl/cpu + + # Ops sidecars not needed for serving smoke; profile prevents startup unless explicitly enabled. + postgres-backup: + profiles: ["ci-skip"] + + backup-offload: + profiles: ["ci-skip"] diff --git a/docker-compose.gpu.yml b/docker-compose.gpu.yml new file mode 100644 index 0000000..99d1a74 --- /dev/null +++ b/docker-compose.gpu.yml @@ -0,0 +1,19 @@ +# GPU overlay for local dev / GPU hosts. Merge with docker-compose.yml: +# docker compose -f docker-compose.yml -f docker-compose.gpu.yml up -d --build +# Or use: make up (auto-detects NVIDIA via nvidia-smi) + +services: + embedding-worker: + gpus: all + environment: + NVIDIA_VISIBLE_DEVICES: all + + go-prediction-api: + gpus: all + environment: + NVIDIA_VISIBLE_DEVICES: all + + trainer-worker: + gpus: all + environment: + NVIDIA_VISIBLE_DEVICES: all diff --git a/docker-compose.yml b/docker-compose.yml index dba2b38..497a8fa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,16 @@ # Integrated stack: Embedding API + GO Prediction API + MLflow + Training API. -# Single public entry: nginx on port 80. Run from repo root: docker compose up --build -# Training API: docker compose --profile training up --build -# Monitoring: docker compose --profile monitoring up +# Single public entry: nginx on port 80. Portable base (CPU-safe; no gpus: here). +# +# Local (recommended): +# make up # auto-adds docker-compose.gpu.yml when NVIDIA is available +# make training-up # --profile training +# make monitoring-up # --profile monitoring +# +# Manual compose: +# docker compose up --build +# docker compose -f docker-compose.yml -f docker-compose.gpu.yml up --build # GPU +# docker compose -f docker-compose.yml -f docker-compose.ci.yml up --build # CI / CPU smoke +# # Copy .env.example to .env and set secrets before first run. networks: proseqgo: @@ -197,11 +206,9 @@ services: working_dir: /app/services/training-api profiles: ["training"] networks: [proseqgo] - gpus: all stop_grace_period: 2h environment: <<: *trainer-env - NVIDIA_VISIBLE_DEVICES: all WORKER_METRICS_PORT: "8001" depends_on: trainer-api: @@ -258,11 +265,9 @@ services: image: proseqgo-embedding-api:local working_dir: /app/services/embedding-api networks: [proseqgo] - gpus: all stop_grace_period: 3700s environment: <<: *embedding-env - NVIDIA_VISIBLE_DEVICES: all WORKER_METRICS_PORT: "8001" # Gives crash-recovery tests a reliable kill window after mark_running. EMBEDDING_JOB_START_DELAY_SEC: ${EMBEDDING_JOB_START_DELAY_SEC:-0} @@ -286,12 +291,10 @@ services: image: proseqgo-go-prediction-api:local working_dir: /app/services/go-prediction-api networks: [proseqgo] - gpus: all environment: PYTHONUNBUFFERED: "1" PYTHONPATH: /app:/app/services/go-prediction-api CAFA_DEVICE: auto - NVIDIA_VISIBLE_DEVICES: all MLFLOW_TRACKING_URI: http://mlflow:5000 MLFLOW_S3_ENDPOINT_URL: ${MLFLOW_S3_ENDPOINT_URL} AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} From 109f5a43d614f04a23be47e7f365f6f736703ab5 Mon Sep 17 00:00:00 2001 From: Behrouz Mirabdi Date: Mon, 27 Jul 2026 15:45:00 +0200 Subject: [PATCH 2/5] feat(make): auto-select GPU overlay and add ci-up/ci-down targets Introduce COMPOSE_DEV_FILES (base + gpu when nvidia-smi works) and COMPOSE_CI_FILES (base + ci overlay) so local dev and CI smoke share one command surface. Add ci-up (ci-env + gateway-auth + compose up) and ci-down (down -v). Route training/monitoring/all-up targets through the same dev file set. Rename up-all/down-all to all-up/all-down for consistency. --- Makefile | 52 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index cc42354..da4fa46 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,24 @@ .PHONY: help up down restart training-up training-down monitoring-up monitoring-down \ - up-all down-all lint test build-images pull-images smoke ci-env gateway-auth + all-up all-down lint test build-images pull-images smoke ci-env gateway-auth ci-up ci-down # Product Python paths linted in CI (Phase 1A). Expand later if needed. LINT_PATHS := src services scripts PYTHON ?= python3 +# Compose file sets (Design A: portable base + optional overlays). +COMPOSE ?= docker compose +COMPOSE_BASE := -f docker-compose.yml +COMPOSE_GPU := -f docker-compose.gpu.yml +COMPOSE_CI := -f docker-compose.ci.yml +# Auto-enable GPU overlay when nvidia-smi works (local dev). +HAS_NVIDIA := $(shell command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi >/dev/null 2>&1 && echo 1) +ifeq ($(HAS_NVIDIA),1) +COMPOSE_DEV_FILES := $(COMPOSE_BASE) $(COMPOSE_GPU) +else +COMPOSE_DEV_FILES := $(COMPOSE_BASE) +endif +COMPOSE_CI_FILES := $(COMPOSE_BASE) $(COMPOSE_CI) + # Image names match docker-compose.yml local tags. EMBEDDING_IMAGE ?= proseqgo-embedding-api:local GO_PRED_IMAGE ?= proseqgo-go-prediction-api:local @@ -26,8 +40,10 @@ GHCR_MLFLOW_IMAGE ?= $(GHCR_REGISTRY)/$(GHCR_OWNER)/proseqgo-mlflow:$(GHCR_TAG) help: @echo "Available targets:" - @echo " make up - Start default services with Docker Compose" - @echo " make down - Stop and remove default Docker Compose services" + @echo " make up - Start default services (GPU overlay if NVIDIA detected)" + @echo " make down - Stop default Docker Compose services" + @echo " make ci-up - Start serving stack for CI/CPU smoke (base + ci overlay)" + @echo " make ci-down - Stop CI stack and remove volumes" @echo " make training-up - Start services with the training profile" @echo " make training-down - Stop services started with the training profile" @echo " make monitoring-up - Start services with the monitoring profile" @@ -41,32 +57,38 @@ help: @echo " make gateway-auth - Write nginx/.htpasswd-* from GATEWAY_* in .env" up: - docker compose up -d --build + $(COMPOSE) $(COMPOSE_DEV_FILES) up -d --build down: - docker compose down + $(COMPOSE) $(COMPOSE_DEV_FILES) down restart: - docker compose down - docker compose up -d --build + $(COMPOSE) $(COMPOSE_DEV_FILES) down + $(COMPOSE) $(COMPOSE_DEV_FILES) up -d --build + +ci-up: ci-env gateway-auth + $(COMPOSE) $(COMPOSE_CI_FILES) up -d --build + +ci-down: + $(COMPOSE) $(COMPOSE_CI_FILES) down -v training-up: - docker compose --profile training up -d --build + $(COMPOSE) $(COMPOSE_DEV_FILES) --profile training up -d --build training-down: - docker compose --profile training down + $(COMPOSE) $(COMPOSE_DEV_FILES) --profile training down monitoring-up: - docker compose --profile monitoring up -d --build + $(COMPOSE) $(COMPOSE_DEV_FILES) --profile monitoring up -d --build monitoring-down: - docker compose --profile monitoring down + $(COMPOSE) $(COMPOSE_DEV_FILES) --profile monitoring down -up-all: - docker compose --profile monitoring --profile training up -d --build +all-up: + $(COMPOSE) $(COMPOSE_DEV_FILES) --profile monitoring --profile training up -d --build -down-all: - docker compose --profile monitoring --profile training down +all-down: + $(COMPOSE) $(COMPOSE_DEV_FILES) --profile monitoring --profile training down # --- CI / quality (same commands locally and in GitHub Actions) --- From bd1843b8ed781a41e9c7ad2125ea91799d12a247 Mon Sep 17 00:00:00 2001 From: Behrouz Mirabdi Date: Mon, 27 Jul 2026 15:47:09 +0200 Subject: [PATCH 3/5] docs(compose): documentation to explain base vs gpu vs ci compose files in README and CI.md. Document make ci-up / make smoke / make ci-down in smoke README. Move gateway vars to the top of .env.example and note CAFA_DEVICE behavior across base, gpu, and ci overlays. --- .github/CI.md | 22 ++++++++++++++++++++-- README.md | 32 ++++++++++++++++++++++++++------ tests/smoke/README.md | 9 +++++++++ 3 files changed, 55 insertions(+), 8 deletions(-) diff --git a/.github/CI.md b/.github/CI.md index 24022e4..ca087af 100644 --- a/.github/CI.md +++ b/.github/CI.md @@ -28,17 +28,35 @@ PR → build only (GHA layer cache, no push) main → build + push sha- and moving main ``` -- Torch images use **CPU wheels** in CI (`TORCH_INDEX_URL=.../cpu`) for smaller/faster builds; local Compose still defaults to CUDA index +- Torch images use **CPU wheels** in CI (`TORCH_INDEX_URL=.../cpu`) for smaller/faster builds; local `make build-images` defaults to CUDA index (`cu132`) - Local build: `make build-images` - Pull published images: `make pull-images` or `GHCR_TAG=sha- make pull-images` - First publish happens after this workflow runs on **`main`**. Packages may be private by default; set package visibility in GitHub Packages if others need to pull. +## Compose overlays (Design A) + +| File | Purpose | +|------|---------| +| `docker-compose.yml` | Portable base (all services; no `gpus:`) | +| `docker-compose.gpu.yml` | Adds `gpus: all` for inference/training workers | +| `docker-compose.ci.yml` | CPU smoke: `CAFA_DEVICE=cpu`, CPU torch build args, skips backup sidecars | + +```bash +make up # base + gpu overlay when nvidia-smi works +make ci-up # base + ci overlay (also runs ci-env + gateway-auth) +make ci-down # tear down CI stack with volumes +``` + ## Compose / secrets in CI (Phase 3) Workflows must never commit real secrets. Pattern: ```bash -make ci-env # copies .env.example → .env if missing +make ci-env # copies .env.example → .env if missing +make gateway-auth # nginx htpasswd from GATEWAY_* in .env +make ci-up # serving stack on CPU +make smoke +make ci-down ``` Use throwaway passwords from `.env.example` only inside ephemeral CI runners. diff --git a/README.md b/README.md index 0b449d2..55dcd61 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,9 @@ CAFA-5-MLOps-solution/ │ ├── streamlit-ui/ # Interactive UI over gateway endpoint │ └── training-api/ # Async train/retrain job API ├── src/ # Core modeling/training/inference modules -├── docker-compose.yml # Full integrated deployment +├── docker-compose.yml # Portable serving stack (CPU-safe base) +├── docker-compose.gpu.yml # GPU overlay (gpus: all for inference workers) +├── docker-compose.ci.yml # CI overlay (CPU torch, CAFA_DEVICE=cpu, trim backups) ├── Makefile # Convenience targets for compose profiles └── README.md ``` @@ -79,16 +81,30 @@ CAFA-5-MLOps-solution/ ### 1) Bring core stack up ```bash -docker compose up --build +make up ``` -or: +`make up` uses the portable base compose file and **automatically adds** `docker-compose.gpu.yml` when `nvidia-smi` is available. On CPU-only hosts the stack still starts (inference uses `CAFA_DEVICE=auto` → CPU). + +Manual compose (equivalent): ```bash -make up +# CPU-only / portable +docker compose up --build + +# GPU host (explicit overlay) +docker compose -f docker-compose.yml -f docker-compose.gpu.yml up --build ``` -Core services started by default: `nginx`, `embedding-api`, `go-prediction-api`, `streamlit-ui`, `mlflow`. +Core services started by default: `nginx`, `embedding-api`, `embedding-worker`, `go-prediction-api`, `streamlit-ui`, `mlflow`. + +**CI / CPU smoke** (Phase 3): + +```bash +make ci-up # base + docker-compose.ci.yml (forces CAFA_DEVICE=cpu, CPU torch wheels) +make smoke +make ci-down +``` ### 2) Bring monitoring up @@ -457,8 +473,10 @@ python scripts/retrain_pipeline.py --config configs/config.yaml \ ## Useful Make Targets ```bash -make up +make up # Default stack (GPU overlay if NVIDIA detected) make down +make ci-up # CPU CI/smoke stack (base + ci overlay) +make ci-down # Stop CI stack and remove volumes make training-up make training-down make monitoring-up @@ -477,6 +495,8 @@ make gateway-auth # Write nginx/.htpasswd-* from GATEWAY_* in .env PR and `main` pushes run **lint**, **unit tests**, and **parallel image builds**. Merges to `main` also **publish to GHCR** (`sha-` + `main`). See [`.github/CI.md`](.github/CI.md). - Registry: **GHCR** (`ghcr.io/behroooz/proseqgo-*`) +- Compose: portable **base** + **`docker-compose.ci.yml`** for CPU smoke (`make ci-up`) +- GPU local dev: **base** + **`docker-compose.gpu.yml`** (`make up` auto-detects NVIDIA) - CI does **not** run training/GPU/retrain jobs - Compose secrets: `.env` from `.env.example`; gateway Basic Auth via `make gateway-auth` (`GATEWAY_ADMIN_*` ≠ `GATEWAY_USER_*`) - Local image rebuild: `make build-images`; pull published: `make pull-images` diff --git a/tests/smoke/README.md b/tests/smoke/README.md index 6112ffd..ff83dcc 100644 --- a/tests/smoke/README.md +++ b/tests/smoke/README.md @@ -7,6 +7,15 @@ # # make gateway-auth # +# Local (GPU if available): +# make up +# make smoke +# +# CPU / CI stack (base + docker-compose.ci.yml): +# make ci-up # also runs ci-env + gateway-auth +# make smoke +# make ci-down +# # From repo root (stack must be up): # ./tests/smoke/smoke_embedding_api.sh # ./tests/smoke/test_embedding_worker_crash_recovery.sh From 83304a21af6758fb1996fecd2e83d2681cac4934 Mon Sep 17 00:00:00 2001 From: Behrouz Mirabdi Date: Mon, 27 Jul 2026 15:47:56 +0200 Subject: [PATCH 4/5] fix(smoke): parse predict-go response reliably in smoke_embedding_api Replace heredoc stdin parsing with python -c + PRED_RESP env var so large JSON responses are validated correctly. Default BASE_URL to http://localhost for consistency with gateway docs. --- tests/smoke/smoke_embedding_api.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/smoke/smoke_embedding_api.sh b/tests/smoke/smoke_embedding_api.sh index 0bcdbbd..a4f48ce 100755 --- a/tests/smoke/smoke_embedding_api.sh +++ b/tests/smoke/smoke_embedding_api.sh @@ -18,7 +18,7 @@ REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" source "${REPO_ROOT}/scripts/load_gateway_env.sh" load_gateway_env "${REPO_ROOT}" -BASE_URL="${BASE_URL:-http://127.0.0.1}" +BASE_URL="${BASE_URL:-http://localhost}" FASTA_EXAMPLE="${REPO_ROOT}/examples/small_sequences.fasta" MAX_FASTA_UPLOAD_BYTES=$((5 * 1024 * 1024)) @@ -111,11 +111,11 @@ PRED_RESP="$(curl "${USER_CURL[@]}" --max-time 1800 -X POST \ -F "fail_fast=true")" echo "$PRED_RESP" -printf '%s' "$PRED_RESP" | python3 <<'PY' -import json -import sys -data = json.load(sys.stdin) +PRED_RESP="$PRED_RESP" python3 -c ' +import json, os +data = json.loads(os.environ["PRED_RESP"]) + assert data["status"] == "succeeded", data results = data["results"] assert len(results) == 2, results @@ -123,7 +123,7 @@ for item in results: assert item.get("sequence_id"), item assert "predictions" in item and isinstance(item["predictions"], list), item print("predict-go-from-fasta OK:", [r["sequence_id"] for r in results]) -PY +' echo "==> FASTA upload too large: expect HTTP 413 (max ${MAX_FASTA_UPLOAD_BYTES} bytes)" LARGE_FASTA="$(mktemp)" From 4bda2299f6f0f4cc825741b35e045b64b8e24f15 Mon Sep 17 00:00:00 2001 From: Behrouz Mirabdi Date: Mon, 27 Jul 2026 15:49:39 +0200 Subject: [PATCH 5/5] style(nginx): nginx basic auth moved to top --- .env.example | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/.env.example b/.env.example index 8e87b71..a9c2e59 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,14 @@ +# NGINX Basic Auth — single source of truth (never commit real .env) +# Generate htpasswd files: make gateway-auth +# Admin ≠ public user (usernames and passwords must both differ). +GATEWAY_ADMIN_USER=admin +GATEWAY_ADMIN_PASSWORD=change-me-gateway-admin +GATEWAY_USER=user +GATEWAY_USER_PASSWORD=change-me-gateway-user + +# Inference device: auto (default in base compose). CI overlay sets CAFA_DEVICE=cpu +# via docker-compose.ci.yml; use docker-compose.gpu.yml + NVIDIA for GPU access. + # Model registry REGISTERED_MODEL_NAME=cafa-go-model PROMOTION_THRESHOLD=0.35 @@ -30,13 +41,4 @@ MLFLOW_ARTIFACT_ROOT=s3://mlflow-artifacts/ # Backup sidecar BACKUP_RETENTION_DAYS=7 BACKUP_RETENTION_WEEKS=4 -BACKUP_OFFLOAD_TARGET=s3://mlflow-db-backups/ - -# NGINX Basic Auth — single source of truth (never commit real .env) -# Generate htpasswd files: make gateway-auth -# Admin ≠ public user (usernames and passwords must both differ). -GATEWAY_ADMIN_USER=admin -GATEWAY_ADMIN_PASSWORD=change-me-gateway-admin -GATEWAY_USER=user -GATEWAY_USER_PASSWORD=change-me-gateway-user - +BACKUP_OFFLOAD_TARGET=s3://mlflow-db-backups/ \ No newline at end of file