Auto prod accept

.env.example

@@ -1,3 +1,14 @@
+# NGINX Basic Auth — single source of truth (never commit real .env)
+# Generate htpasswd files: make gateway-auth
+# Admin ≠ public user (usernames and passwords must both differ).
+GATEWAY_ADMIN_USER=admin
+GATEWAY_ADMIN_PASSWORD=change-me-gateway-admin
+GATEWAY_USER=user
+GATEWAY_USER_PASSWORD=change-me-gateway-user
+
+# Inference device: auto (default in base compose). CI overlay sets CAFA_DEVICE=cpu
+# via docker-compose.ci.yml; use docker-compose.gpu.yml + NVIDIA for GPU access.
+
 # Model registry
 REGISTERED_MODEL_NAME=cafa-go-model
 PROMOTION_THRESHOLD=0.35
@@ -30,13 +41,4 @@ MLFLOW_ARTIFACT_ROOT=s3://mlflow-artifacts/
 # Backup sidecar
 BACKUP_RETENTION_DAYS=7
 BACKUP_RETENTION_WEEKS=4
-BACKUP_OFFLOAD_TARGET=s3://mlflow-db-backups/
-
-# NGINX Basic Auth — single source of truth (never commit real .env)
-# Generate htpasswd files: make gateway-auth
-# Admin ≠ public user (usernames and passwords must both differ).
-GATEWAY_ADMIN_USER=admin
-GATEWAY_ADMIN_PASSWORD=change-me-gateway-admin
-GATEWAY_USER=user
-GATEWAY_USER_PASSWORD=change-me-gateway-user
-
+BACKUP_OFFLOAD_TARGET=s3://mlflow-db-backups/

.github/CI.md

@@ -28,17 +28,35 @@ PR     → build only (GHA layer cache, no push)
 main   → build + push sha-<fullsha> and moving main
 ```
 
-- Torch images use **CPU wheels** in CI (`TORCH_INDEX_URL=.../cpu`) for smaller/faster builds; local Compose still defaults to CUDA index
+- Torch images use **CPU wheels** in CI (`TORCH_INDEX_URL=.../cpu`) for smaller/faster builds; local `make build-images` defaults to CUDA index (`cu132`)
 - Local build: `make build-images`
 - Pull published images: `make pull-images` or `GHCR_TAG=sha-<fullsha> make pull-images`
 - First publish happens after this workflow runs on **`main`**. Packages may be private by default; set package visibility in GitHub Packages if others need to pull.
 
+## Compose overlays (Design A)
+
+| File | Purpose |
+|------|---------|
+| `docker-compose.yml` | Portable base (all services; no `gpus:`) |
+| `docker-compose.gpu.yml` | Adds `gpus: all` for inference/training workers |
+| `docker-compose.ci.yml` | CPU smoke: `CAFA_DEVICE=cpu`, CPU torch build args, skips backup sidecars |
+
+```bash
+make up        # base + gpu overlay when nvidia-smi works
+make ci-up     # base + ci overlay (also runs ci-env + gateway-auth)
+make ci-down   # tear down CI stack with volumes
+```
+
 ## Compose / secrets in CI (Phase 3)
 
 Workflows must never commit real secrets. Pattern:
 
 ```bash
-make ci-env   # copies .env.example → .env if missing
+make ci-env        # copies .env.example → .env if missing
+make gateway-auth  # nginx htpasswd from GATEWAY_* in .env
+make ci-up         # serving stack on CPU
+make smoke
+make ci-down
 ```
 
 Use throwaway passwords from `.env.example` only inside ephemeral CI runners.

Makefile

@@ -1,10 +1,24 @@
 .PHONY: help up down restart training-up training-down monitoring-up monitoring-down \
-	up-all down-all lint test build-images pull-images smoke ci-env gateway-auth
+	all-up all-down lint test build-images pull-images smoke ci-env gateway-auth ci-up ci-down
 
 # Product Python paths linted in CI (Phase 1A). Expand later if needed.
 LINT_PATHS := src services scripts
 PYTHON ?= python3
 
+# Compose file sets (Design A: portable base + optional overlays).
+COMPOSE ?= docker compose
+COMPOSE_BASE := -f docker-compose.yml
+COMPOSE_GPU := -f docker-compose.gpu.yml
+COMPOSE_CI := -f docker-compose.ci.yml
+# Auto-enable GPU overlay when nvidia-smi works (local dev).
+HAS_NVIDIA := $(shell command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi >/dev/null 2>&1 && echo 1)
+ifeq ($(HAS_NVIDIA),1)
+COMPOSE_DEV_FILES := $(COMPOSE_BASE) $(COMPOSE_GPU)
+else
+COMPOSE_DEV_FILES := $(COMPOSE_BASE)
+endif
+COMPOSE_CI_FILES := $(COMPOSE_BASE) $(COMPOSE_CI)
+
 # Image names match docker-compose.yml local tags.
 EMBEDDING_IMAGE ?= proseqgo-embedding-api:local
 GO_PRED_IMAGE ?= proseqgo-go-prediction-api:local
@@ -26,8 +40,10 @@ GHCR_MLFLOW_IMAGE ?= $(GHCR_REGISTRY)/$(GHCR_OWNER)/proseqgo-mlflow:$(GHCR_TAG)
 
 help:
 	@echo "Available targets:"
-	@echo "  make up               - Start default services with Docker Compose"
-	@echo "  make down             - Stop and remove default Docker Compose services"
+	@echo "  make up               - Start default services (GPU overlay if NVIDIA detected)"
+	@echo "  make down             - Stop default Docker Compose services"
+	@echo "  make ci-up            - Start serving stack for CI/CPU smoke (base + ci overlay)"
+	@echo "  make ci-down          - Stop CI stack and remove volumes"
 	@echo "  make training-up      - Start services with the training profile"
 	@echo "  make training-down    - Stop services started with the training profile"
 	@echo "  make monitoring-up    - Start services with the monitoring profile"
@@ -41,32 +57,38 @@ help:
 	@echo "  make gateway-auth     - Write nginx/.htpasswd-* from GATEWAY_* in .env"
 
 up:
-	docker compose up -d --build
+	$(COMPOSE) $(COMPOSE_DEV_FILES) up -d --build
 
 down:
-	docker compose down
+	$(COMPOSE) $(COMPOSE_DEV_FILES) down
 
 restart:
-	docker compose down
-	docker compose up -d --build
+	$(COMPOSE) $(COMPOSE_DEV_FILES) down
+	$(COMPOSE) $(COMPOSE_DEV_FILES) up -d --build
+
+ci-up: ci-env gateway-auth
+	$(COMPOSE) $(COMPOSE_CI_FILES) up -d --build
+
+ci-down:
+	$(COMPOSE) $(COMPOSE_CI_FILES) down -v
 
 training-up:
-	docker compose --profile training up -d --build
+	$(COMPOSE) $(COMPOSE_DEV_FILES) --profile training up -d --build
 
 training-down:
-	docker compose --profile training down
+	$(COMPOSE) $(COMPOSE_DEV_FILES) --profile training down
 
 monitoring-up:
-	docker compose --profile monitoring up -d --build
+	$(COMPOSE) $(COMPOSE_DEV_FILES) --profile monitoring up -d --build
 
 monitoring-down:
-	docker compose --profile monitoring down
+	$(COMPOSE) $(COMPOSE_DEV_FILES) --profile monitoring down
 
-up-all:
-	docker compose --profile monitoring --profile training up -d --build
+all-up:
+	$(COMPOSE) $(COMPOSE_DEV_FILES) --profile monitoring --profile training up -d --build
 
-down-all:
-	docker compose --profile monitoring --profile training down
+all-down:
+	$(COMPOSE) $(COMPOSE_DEV_FILES) --profile monitoring --profile training down
 
 # --- CI / quality (same commands locally and in GitHub Actions) ---
 

README.md

@@ -69,7 +69,9 @@ CAFA-5-MLOps-solution/
 │   ├── streamlit-ui/             # Interactive UI over gateway endpoint
 │   └── training-api/             # Async train/retrain job API
 ├── src/                          # Core modeling/training/inference modules
-├── docker-compose.yml            # Full integrated deployment
+├── docker-compose.yml            # Portable serving stack (CPU-safe base)
+├── docker-compose.gpu.yml        # GPU overlay (gpus: all for inference workers)
+├── docker-compose.ci.yml         # CI overlay (CPU torch, CAFA_DEVICE=cpu, trim backups)
 ├── Makefile                      # Convenience targets for compose profiles
 └── README.md
 ```
@@ -79,16 +81,30 @@ CAFA-5-MLOps-solution/
 ### 1) Bring core stack up
 
 ```bash
-docker compose up --build
+make up
 ```
 
-or:
+`make up` uses the portable base compose file and **automatically adds** `docker-compose.gpu.yml` when `nvidia-smi` is available. On CPU-only hosts the stack still starts (inference uses `CAFA_DEVICE=auto` → CPU).
+
+Manual compose (equivalent):
 
 ```bash
-make up
+# CPU-only / portable
+docker compose up --build
+
+# GPU host (explicit overlay)
+docker compose -f docker-compose.yml -f docker-compose.gpu.yml up --build
 ```
 
-Core services started by default: `nginx`, `embedding-api`, `go-prediction-api`, `streamlit-ui`, `mlflow`.
+Core services started by default: `nginx`, `embedding-api`, `embedding-worker`, `go-prediction-api`, `streamlit-ui`, `mlflow`.
+
+**CI / CPU smoke** (Phase 3):
+
+```bash
+make ci-up    # base + docker-compose.ci.yml (forces CAFA_DEVICE=cpu, CPU torch wheels)
+make smoke
+make ci-down
+```
 
 ### 2) Bring monitoring up
 
@@ -457,8 +473,10 @@ python scripts/retrain_pipeline.py --config configs/config.yaml \
 ## Useful Make Targets
 
 ```bash
-make up
+make up              # Default stack (GPU overlay if NVIDIA detected)
 make down
+make ci-up           # CPU CI/smoke stack (base + ci overlay)
+make ci-down         # Stop CI stack and remove volumes
 make training-up
 make training-down
 make monitoring-up
@@ -477,6 +495,8 @@ make gateway-auth    # Write nginx/.htpasswd-* from GATEWAY_* in .env
 PR and `main` pushes run **lint**, **unit tests**, and **parallel image builds**. Merges to `main` also **publish to GHCR** (`sha-<fullsha>` + `main`). See [`.github/CI.md`](.github/CI.md).
 
 - Registry: **GHCR** (`ghcr.io/behroooz/proseqgo-*`)
+- Compose: portable **base** + **`docker-compose.ci.yml`** for CPU smoke (`make ci-up`)
+- GPU local dev: **base** + **`docker-compose.gpu.yml`** (`make up` auto-detects NVIDIA)
 - CI does **not** run training/GPU/retrain jobs
 - Compose secrets: `.env` from `.env.example`; gateway Basic Auth via `make gateway-auth` (`GATEWAY_ADMIN_*` ≠ `GATEWAY_USER_*`)
 - Local image rebuild: `make build-images`; pull published: `make pull-images`

docker-compose.ci.yml

@@ -0,0 +1,32 @@
+# CI / CPU smoke overlay. Merge with docker-compose.yml:
+#   docker compose -f docker-compose.yml -f docker-compose.ci.yml up -d --build
+# Or use: make ci-up
+
+services:
+  embedding-api:
+    environment:
+      CAFA_DEVICE: cpu
+    build:
+      args:
+        TORCH_INDEX_URL: https://download.pytorch.org/whl/cpu
+
+  embedding-worker:
+    environment:
+      CAFA_DEVICE: cpu
+    build:
+      args:
+        TORCH_INDEX_URL: https://download.pytorch.org/whl/cpu
+
+  go-prediction-api:
+    environment:
+      CAFA_DEVICE: cpu
+    build:
+      args:
+        TORCH_INDEX_URL: https://download.pytorch.org/whl/cpu
+
+  # Ops sidecars not needed for serving smoke; profile prevents startup unless explicitly enabled.
+  postgres-backup:
+    profiles: ["ci-skip"]
+
+  backup-offload:
+    profiles: ["ci-skip"]

docker-compose.gpu.yml

@@ -0,0 +1,19 @@
+# GPU overlay for local dev / GPU hosts. Merge with docker-compose.yml:
+#   docker compose -f docker-compose.yml -f docker-compose.gpu.yml up -d --build
+# Or use: make up (auto-detects NVIDIA via nvidia-smi)
+
+services:
+  embedding-worker:
+    gpus: all
+    environment:
+      NVIDIA_VISIBLE_DEVICES: all
+
+  go-prediction-api:
+    gpus: all
+    environment:
+      NVIDIA_VISIBLE_DEVICES: all
+
+  trainer-worker:
+    gpus: all
+    environment:
+      NVIDIA_VISIBLE_DEVICES: all

docker-compose.yml

@@ -1,7 +1,16 @@
 # Integrated stack: Embedding API + GO Prediction API + MLflow + Training API.
-# Single public entry: nginx on port 80. Run from repo root: docker compose up --build
-# Training API: docker compose --profile training up --build
-# Monitoring: docker compose --profile monitoring up
+# Single public entry: nginx on port 80. Portable base (CPU-safe; no gpus: here).
+#
+# Local (recommended):
+#   make up              # auto-adds docker-compose.gpu.yml when NVIDIA is available
+#   make training-up     # --profile training
+#   make monitoring-up   # --profile monitoring
+#
+# Manual compose:
+#   docker compose up --build
+#   docker compose -f docker-compose.yml -f docker-compose.gpu.yml up --build   # GPU
+#   docker compose -f docker-compose.yml -f docker-compose.ci.yml up --build    # CI / CPU smoke
+#
 # Copy .env.example to .env and set secrets before first run.
 networks:
   proseqgo:
@@ -197,11 +206,9 @@ services:
     working_dir: /app/services/training-api
     profiles: ["training"]
     networks: [proseqgo]
-    gpus: all
     stop_grace_period: 2h
     environment:
       <<: *trainer-env
-      NVIDIA_VISIBLE_DEVICES: all
       WORKER_METRICS_PORT: "8001"
     depends_on:
       trainer-api:
@@ -258,11 +265,9 @@ services:
     image: proseqgo-embedding-api:local
     working_dir: /app/services/embedding-api
     networks: [proseqgo]
-    gpus: all
     stop_grace_period: 3700s
     environment:
       <<: *embedding-env
-      NVIDIA_VISIBLE_DEVICES: all
       WORKER_METRICS_PORT: "8001"
       # Gives crash-recovery tests a reliable kill window after mark_running.
       EMBEDDING_JOB_START_DELAY_SEC: ${EMBEDDING_JOB_START_DELAY_SEC:-0}
@@ -286,12 +291,10 @@ services:
     image: proseqgo-go-prediction-api:local
     working_dir: /app/services/go-prediction-api
     networks: [proseqgo]
-    gpus: all
     environment:
       PYTHONUNBUFFERED: "1"
       PYTHONPATH: /app:/app/services/go-prediction-api
       CAFA_DEVICE: auto
-      NVIDIA_VISIBLE_DEVICES: all
       MLFLOW_TRACKING_URI: http://mlflow:5000
       MLFLOW_S3_ENDPOINT_URL: ${MLFLOW_S3_ENDPOINT_URL}
       AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}

tests/smoke/README.md

@@ -7,6 +7,15 @@
 #
 #   make gateway-auth
 #
+# Local (GPU if available):
+#   make up
+#   make smoke
+#
+# CPU / CI stack (base + docker-compose.ci.yml):
+#   make ci-up      # also runs ci-env + gateway-auth
+#   make smoke
+#   make ci-down
+#
 # From repo root (stack must be up):
 #   ./tests/smoke/smoke_embedding_api.sh
 #   ./tests/smoke/test_embedding_worker_crash_recovery.sh

tests/smoke/smoke_embedding_api.sh

@@ -18,7 +18,7 @@ REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
 source "${REPO_ROOT}/scripts/load_gateway_env.sh"
 load_gateway_env "${REPO_ROOT}"
 
-BASE_URL="${BASE_URL:-http://127.0.0.1}"
+BASE_URL="${BASE_URL:-http://localhost}"
 FASTA_EXAMPLE="${REPO_ROOT}/examples/small_sequences.fasta"
 MAX_FASTA_UPLOAD_BYTES=$((5 * 1024 * 1024))
 
@@ -111,19 +111,19 @@ PRED_RESP="$(curl "${USER_CURL[@]}" --max-time 1800 -X POST \
   -F "fail_fast=true")"
 echo "$PRED_RESP"
 
-printf '%s' "$PRED_RESP" | python3 <<'PY'
-import json
-import sys
 
-data = json.load(sys.stdin)
+PRED_RESP="$PRED_RESP" python3 -c '
+import json, os
+data = json.loads(os.environ["PRED_RESP"])
+
 assert data["status"] == "succeeded", data
 results = data["results"]
 assert len(results) == 2, results
 for item in results:
     assert item.get("sequence_id"), item
     assert "predictions" in item and isinstance(item["predictions"], list), item
 print("predict-go-from-fasta OK:", [r["sequence_id"] for r in results])
-PY
+'
 
 echo "==> FASTA upload too large: expect HTTP 413 (max ${MAX_FASTA_UPLOAD_BYTES} bytes)"
 LARGE_FASTA="$(mktemp)"