diff --git a/.env.example b/.env.example index 75e55bd..8e87b71 100644 --- a/.env.example +++ b/.env.example @@ -31,3 +31,12 @@ MLFLOW_ARTIFACT_ROOT=s3://mlflow-artifacts/ BACKUP_RETENTION_DAYS=7 BACKUP_RETENTION_WEEKS=4 BACKUP_OFFLOAD_TARGET=s3://mlflow-db-backups/ + +# NGINX Basic Auth — single source of truth (never commit real .env) +# Generate htpasswd files: make gateway-auth +# Admin ≠ public user (usernames and passwords must both differ). +GATEWAY_ADMIN_USER=admin +GATEWAY_ADMIN_PASSWORD=change-me-gateway-admin +GATEWAY_USER=user +GATEWAY_USER_PASSWORD=change-me-gateway-user + diff --git a/Makefile b/Makefile index b7ed0a0..cc42354 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .PHONY: help up down restart training-up training-down monitoring-up monitoring-down \ - up-all down-all lint test build-images pull-images smoke ci-env + up-all down-all lint test build-images pull-images smoke ci-env gateway-auth # Product Python paths linted in CI (Phase 1A). Expand later if needed. LINT_PATHS := src services scripts @@ -37,7 +37,8 @@ help: @echo " make build-images - Build all product Docker images" @echo " make pull-images - Pull product images from GHCR (GHCR_TAG=main|sha-...)" @echo " make smoke - Run Compose smoke scripts (stack must be up)" - @echo " make ci-env - Copy .env.example -> .env for local/CI Compose" + @echo " make ci-env - Copy .env.example -> .env if missing" + @echo " make gateway-auth - Write nginx/.htpasswd-* from GATEWAY_* in .env" up: docker compose up -d --build @@ -78,6 +79,9 @@ ci-env: echo "Created .env from .env.example"; \ fi +gateway-auth: ci-env + ./scripts/prepare_gateway_auth.sh + lint: ruff check $(LINT_PATHS) diff --git a/README.md b/README.md index 487c44d..0b449d2 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# CAFA-5 MLOps Solution +# ProSeqGO: Protein Sequence Gene Onthology prediction -End-to-end MLOps platform for CAFA-5 protein function prediction (sequence -> embedding -> GO terms), with model lifecycle management, secured gateway routing, and production-oriented monitoring. +End-to-end MLOps platform for protein function prediction (sequence -> embedding -> GO terms), with model lifecycle management, secured gateway routing, and production-oriented monitoring. ## Problem This Project Solves @@ -469,6 +469,7 @@ make build-images # Build all five product images make pull-images # Pull product images from GHCR (GHCR_TAG=main|sha-...) make smoke # Smoke scripts (Compose stack must already be up) make ci-env # Copy .env.example → .env if missing +make gateway-auth # Write nginx/.htpasswd-* from GATEWAY_* in .env ``` ## CI (GitHub Actions) @@ -477,7 +478,7 @@ PR and `main` pushes run **lint**, **unit tests**, and **parallel image builds** - Registry: **GHCR** (`ghcr.io/behroooz/proseqgo-*`) - CI does **not** run training/GPU/retrain jobs -- Compose in CI will use `make ci-env` (`.env.example` only)—never commit real secrets +- Compose secrets: `.env` from `.env.example`; gateway Basic Auth via `make gateway-auth` (`GATEWAY_ADMIN_*` ≠ `GATEWAY_USER_*`) - Local image rebuild: `make build-images`; pull published: `make pull-images` ## Service-Specific Documentation diff --git a/docker-compose.yml b/docker-compose.yml index a3d8c35..dba2b38 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -315,6 +315,9 @@ services: networks: [proseqgo] environment: GATEWAY_BASE_URL: http://nginx + # Predict-route credentials (public user). Must match make gateway-auth / .htpasswd-user. + GATEWAY_USER: ${GATEWAY_USER:-user} + GATEWAY_USER_PASSWORD: ${GATEWAY_USER_PASSWORD:-change-me-gateway-user} depends_on: - embedding-api - go-prediction-api diff --git a/nginx/README.md b/nginx/README.md index 2926cd8..6d6ec5e 100644 --- a/nginx/README.md +++ b/nginx/README.md @@ -15,6 +15,7 @@ This milestone introduced NGINX as the single ingress gateway for the CAFA-5 MLO - **Authentication segmentation:** - Admin-only access for embedding endpoints and MLflow (`.htpasswd-admin`). - User-level access for prediction endpoints (`.htpasswd-user`). + - Local discipline: set distinct `GATEWAY_ADMIN_*` and `GATEWAY_USER_*` in `.env`, then `make gateway-auth`. - **Route-level rate limiting:** Separate request budgets for admin and prediction paths with `429` on limit exceed. - **Route-specific payload limits:** Enforced `client_max_body_size` per endpoint group to protect upstream services. - **Hardened proxy timeouts:** Long read/send timeouts for model/training workloads; bounded connect timeout. diff --git a/scripts/load_gateway_env.sh b/scripts/load_gateway_env.sh new file mode 100755 index 0000000..501368d --- /dev/null +++ b/scripts/load_gateway_env.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Load GATEWAY_* from repo .env into the current shell without executing arbitrary lines. +# +# Usage: +# source scripts/load_gateway_env.sh +# load_gateway_env /path/to/repo/root + +load_gateway_env() { + local repo_root="${1:-.}" + local env_file="${repo_root}/.env" + local key val line + + if [[ -f "${env_file}" ]]; then + while IFS= read -r line || [[ -n "${line}" ]]; do + [[ -z "${line}" || "${line}" =~ ^[[:space:]]*# ]] && continue + case "${line}" in + GATEWAY_*=*) + key="${line%%=*}" + val="${line#*=}" + if [[ "${val}" =~ ^\".*\"$ || "${val}" =~ ^\'.*\'$ ]]; then + val="${val:1:-1}" + fi + if [[ -z "${!key:-}" ]]; then + printf -v "${key}" '%s' "${val}" + export "${key?}" + fi + ;; + esac + done <"${env_file}" + fi + + export GATEWAY_ADMIN_USER="${GATEWAY_ADMIN_USER:-admin}" + export GATEWAY_ADMIN_PASSWORD="${GATEWAY_ADMIN_PASSWORD:-change-me-gateway-admin}" + export GATEWAY_USER="${GATEWAY_USER:-user}" + export GATEWAY_USER_PASSWORD="${GATEWAY_USER_PASSWORD:-change-me-gateway-user}" +} diff --git a/scripts/prepare_gateway_auth.sh b/scripts/prepare_gateway_auth.sh new file mode 100755 index 0000000..977baa4 --- /dev/null +++ b/scripts/prepare_gateway_auth.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Generate nginx Basic Auth files from .env (GATEWAY_*). +# Files are gitignored (nginx/.htpasswd-*). Never commit real passwords. +# +# GATEWAY_ADMIN_USER / GATEWAY_ADMIN_PASSWORD → .htpasswd-admin +# ( /api/v1/* admin routes, /mlflow, /api/train ) +# GATEWAY_USER / GATEWAY_USER_PASSWORD → .htpasswd-user +# ( /api/predict/*, predict-go-* ) +# +# Admin and public-user passwords must differ. + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=load_gateway_env.sh +source "${REPO_ROOT}/scripts/load_gateway_env.sh" +load_gateway_env "${REPO_ROOT}" + +NGINX_DIR="${REPO_ROOT}/nginx" + +if ! command -v htpasswd >/dev/null 2>&1; then + echo "htpasswd not found. Install apache2-utils (Debian/Ubuntu) or httpd-tools (RHEL)." >&2 + exit 1 +fi + +if [[ -z "${GATEWAY_ADMIN_PASSWORD}" || -z "${GATEWAY_USER_PASSWORD}" ]]; then + echo "GATEWAY_ADMIN_PASSWORD and GATEWAY_USER_PASSWORD must be set (see .env.example)." >&2 + exit 1 +fi + +if [[ "${GATEWAY_ADMIN_PASSWORD}" == "${GATEWAY_USER_PASSWORD}" ]]; then + echo "Refusing to write htpasswd: GATEWAY_ADMIN_PASSWORD must differ from GATEWAY_USER_PASSWORD." >&2 + exit 1 +fi + +if [[ "${GATEWAY_ADMIN_USER}" == "${GATEWAY_USER}" ]]; then + echo "Refusing to write htpasswd: GATEWAY_ADMIN_USER must differ from GATEWAY_USER." >&2 + exit 1 +fi + +mkdir -p "${NGINX_DIR}" +htpasswd -nbB "${GATEWAY_ADMIN_USER}" "${GATEWAY_ADMIN_PASSWORD}" >"${NGINX_DIR}/.htpasswd-admin" +htpasswd -nbB "${GATEWAY_USER}" "${GATEWAY_USER_PASSWORD}" >"${NGINX_DIR}/.htpasswd-user" + +echo "Wrote ${NGINX_DIR}/.htpasswd-admin and ${NGINX_DIR}/.htpasswd-user" +echo "Admin (ops): ${GATEWAY_ADMIN_USER}" +echo "User (predict): ${GATEWAY_USER}" +echo "Next: recreate streamlit if needed → docker compose up -d streamlit-ui" +echo "UI should use the predict user; admin password is for /mlflow and admin APIs." diff --git a/services/streamlit-ui/README.md b/services/streamlit-ui/README.md index fe50a7d..fd92449 100644 --- a/services/streamlit-ui/README.md +++ b/services/streamlit-ui/README.md @@ -16,7 +16,7 @@ The Streamlit app in `services/streamlit-ui/app.py` includes: - protein sequence input, - `top_k` input (`1..500`), - gateway base URL input (defaults from `GATEWAY_BASE_URL` env var), - - API username/password fields, + - API username/password fields (defaults from `GATEWAY_USER` / `GATEWAY_USER_PASSWORD`), - optional TLS verification toggle. - Input QC and validation: - trims whitespace/newlines from sequence, @@ -100,7 +100,8 @@ Important auth boundary: - Streamlit UI path itself is public (`/ui/`). - Prediction endpoint `/api/v1/predict-go-from-sequences` still uses NGINX basic auth (`.htpasswd-user`). -- Credentials are provided by user in the Streamlit form and sent with each API request. +- Defaults come from `.env` via Compose (`GATEWAY_USER` / `GATEWAY_USER_PASSWORD`). Run `make gateway-auth` so htpasswd matches. +- Admin credentials (`GATEWAY_ADMIN_*`) are for `/mlflow` and admin `/api/v1` routes — not the public UI defaults. ## Run And Access diff --git a/services/streamlit-ui/app.py b/services/streamlit-ui/app.py index 080dede..7c41633 100644 --- a/services/streamlit-ui/app.py +++ b/services/streamlit-ui/app.py @@ -24,6 +24,9 @@ "prediction APIs through the NGINX gateway." ) DEFAULT_GATEWAY_URL = os.getenv("GATEWAY_BASE_URL", "http://localhost") +# Public predict-route defaults (must match nginx/.htpasswd-user from make gateway-auth). +DEFAULT_API_USERNAME = os.getenv("GATEWAY_USER", "") +DEFAULT_API_PASSWORD = os.getenv("GATEWAY_USER_PASSWORD", "") PREDICT_SEQUENCES_ENDPOINT = "/api/v1/predict-go-from-sequences" PREDICT_FASTA_ENDPOINT = "/api/v1/predict-go-from-fasta" MAX_TOP_K = 500 @@ -184,8 +187,8 @@ def _render_shared_connection_fields( ) -> tuple[str, str, str, bool, int]: top_k = st.number_input("top_k", min_value=1, max_value=MAX_TOP_K, value=10, step=1) gateway_base_url_input = st.text_input("Gateway base URL", value=gateway_base_url) - username = st.text_input("API username") - password = st.text_input("API password", type="password") + username = st.text_input("API username", value=DEFAULT_API_USERNAME) + password = st.text_input("API password", type="password", value=DEFAULT_API_PASSWORD) verify_tls = st.checkbox("Verify TLS", value=verify_tls_default) return gateway_base_url_input, username, password, verify_tls, int(top_k) diff --git a/tests/smoke/README.md b/tests/smoke/README.md index 9efdd65..6112ffd 100644 --- a/tests/smoke/README.md +++ b/tests/smoke/README.md @@ -2,7 +2,14 @@ # # These are not unit tests: they expect docker compose services to be up. # -# From repo root: +# Gateway credentials live in `.env` (`GATEWAY_ADMIN_*`, `GATEWAY_USER_*`). +# Sync nginx htpasswd (admin ≠ public user): +# +# make gateway-auth +# +# From repo root (stack must be up): # ./tests/smoke/smoke_embedding_api.sh # ./tests/smoke/test_embedding_worker_crash_recovery.sh # MLFLOW_TRACKING_URI=http://127.0.0.1/mlflow python tests/smoke/mlflow_smoke_test.py +# +# Admin credentials are used for /api/v1/jobs*; public user for predict-go-*. diff --git a/tests/smoke/smoke_embedding_api.sh b/tests/smoke/smoke_embedding_api.sh index d035a9c..0bcdbbd 100755 --- a/tests/smoke/smoke_embedding_api.sh +++ b/tests/smoke/smoke_embedding_api.sh @@ -1,33 +1,45 @@ #!/usr/bin/env bash # Smoke test for the Embedding API (direct uvicorn or via nginx gateway on port 80). # Usage (from repo root): +# make gateway-auth # once: sync nginx htpasswd from .env # ./tests/smoke/smoke_embedding_api.sh # BASE_URL=http://127.0.0.1:8000 ./tests/smoke/smoke_embedding_api.sh -# CURL_INSECURE=1 API_USER=user API_PASS=secret \ -# BASE_URL=https://localhost ./tests/smoke/smoke_embedding_api.sh +# +# Credentials come from .env: +# GATEWAY_ADMIN_* → /api/v1/health, /api/v1/jobs, artifacts +# GATEWAY_USER_* → /api/v1/predict-go-from-fasta +# Overrides: ADMIN_USER/ADMIN_PASS, API_USER/API_PASS (predict user). set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +# shellcheck source=../../scripts/load_gateway_env.sh +source "${REPO_ROOT}/scripts/load_gateway_env.sh" +load_gateway_env "${REPO_ROOT}" + BASE_URL="${BASE_URL:-http://127.0.0.1}" FASTA_EXAMPLE="${REPO_ROOT}/examples/small_sequences.fasta" MAX_FASTA_UPLOAD_BYTES=$((5 * 1024 * 1024)) -CURL_OPTS=(-sS) +ADMIN_USER="${ADMIN_USER:-${GATEWAY_ADMIN_USER}}" +ADMIN_PASS="${ADMIN_PASS:-${GATEWAY_ADMIN_PASSWORD}}" +API_USER="${API_USER:-${GATEWAY_USER}}" +API_PASS="${API_PASS:-${GATEWAY_USER_PASSWORD}}" + +CURL_BASE=(-sS) if [[ -n "${CURL_INSECURE:-}" ]]; then - CURL_OPTS+=(-k) -fi -if [[ -n "${API_USER:-}" && -n "${API_PASS:-}" ]]; then - CURL_OPTS+=(-u "${API_USER}:${API_PASS}") + CURL_BASE+=(-k) fi +ADMIN_CURL=("${CURL_BASE[@]}" -u "${ADMIN_USER}:${ADMIN_PASS}") +USER_CURL=("${CURL_BASE[@]}" -u "${API_USER}:${API_PASS}") -echo "==> Health: GET ${BASE_URL}/api/v1/health" -curl "${CURL_OPTS[@]}" "${BASE_URL}/api/v1/health" +echo "==> Health: GET ${BASE_URL}/api/v1/health (admin)" +curl "${ADMIN_CURL[@]}" "${BASE_URL}/api/v1/health" echo -echo "==> Submit job: POST ${BASE_URL}/api/v1/jobs" -RESP="$(curl "${CURL_OPTS[@]}" -X POST "${BASE_URL}/api/v1/jobs" \ +echo "==> Submit job: POST ${BASE_URL}/api/v1/jobs (admin)" +RESP="$(curl "${ADMIN_CURL[@]}" -X POST "${BASE_URL}/api/v1/jobs" \ -H "Content-Type: application/json" \ -d '{ "stage": "test", @@ -47,7 +59,7 @@ echo "==> Job ID: ${JOB_ID}" echo "==> Poll until succeeded (max ~120s)" for _ in $(seq 1 60); do - ST="$(curl "${CURL_OPTS[@]}" "${BASE_URL}/api/v1/jobs/${JOB_ID}")" + ST="$(curl "${ADMIN_CURL[@]}" "${BASE_URL}/api/v1/jobs/${JOB_ID}")" STATUS="$(printf '%s' "$ST" | python3 -c "import sys, json; print(json.load(sys.stdin)['status'])")" if [[ "$STATUS" == "succeeded" ]]; then echo "$ST" | python3 -m json.tool @@ -67,9 +79,9 @@ fi OUT_DIR="$(mktemp -d)" echo "==> Download artifacts to ${OUT_DIR}" -curl "${CURL_OPTS[@]}" -o "${OUT_DIR}/test_ids.npy" \ +curl "${ADMIN_CURL[@]}" -o "${OUT_DIR}/test_ids.npy" \ "${BASE_URL}/api/v1/jobs/${JOB_ID}/artifacts/test_ids.npy" -curl "${CURL_OPTS[@]}" -o "${OUT_DIR}/test_embeddings.npy" \ +curl "${ADMIN_CURL[@]}" -o "${OUT_DIR}/test_embeddings.npy" \ "${BASE_URL}/api/v1/jobs/${JOB_ID}/artifacts/test_embeddings.npy" echo "==> Verify shapes (expect N=2, D=1280 for esm2, float32)" @@ -87,8 +99,8 @@ assert str(emb.dtype) == "float32" print("OK") PY -echo "==> Predict GO from FASTA: POST ${BASE_URL}/api/v1/predict-go-from-fasta" -PRED_RESP="$(curl "${CURL_OPTS[@]}" --max-time 1800 -X POST \ +echo "==> Predict GO from FASTA: POST ${BASE_URL}/api/v1/predict-go-from-fasta (user)" +PRED_RESP="$(curl "${USER_CURL[@]}" --max-time 1800 -X POST \ "${BASE_URL}/api/v1/predict-go-from-fasta" \ -F "fasta_file=@${FASTA_EXAMPLE}" \ -F "backend=esm2" \ @@ -116,7 +128,7 @@ PY echo "==> FASTA upload too large: expect HTTP 413 (max ${MAX_FASTA_UPLOAD_BYTES} bytes)" LARGE_FASTA="$(mktemp)" python3 -c "import sys; sys.stdout.buffer.write(b'x' * (${MAX_FASTA_UPLOAD_BYTES} + 1))" >"${LARGE_FASTA}" -HTTP_CODE="$(curl "${CURL_OPTS[@]}" -o /dev/null -w "%{http_code}" -X POST \ +HTTP_CODE="$(curl "${USER_CURL[@]}" -o /dev/null -w "%{http_code}" -X POST \ "${BASE_URL}/api/v1/predict-go-from-fasta" \ -F "fasta_file=@${LARGE_FASTA};type=text/plain" \ -F "backend=esm2")"