Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -31,3 +31,12 @@ MLFLOW_ARTIFACT_ROOT=s3://mlflow-artifacts/
BACKUP_RETENTION_DAYS=7
BACKUP_RETENTION_WEEKS=4
BACKUP_OFFLOAD_TARGET=s3://mlflow-db-backups/

# NGINX Basic Auth — single source of truth (never commit real .env)
# Generate htpasswd files: make gateway-auth
# Admin ≠ public user (usernames and passwords must both differ).
GATEWAY_ADMIN_USER=admin
GATEWAY_ADMIN_PASSWORD=change-me-gateway-admin
GATEWAY_USER=user
GATEWAY_USER_PASSWORD=change-me-gateway-user

8 changes: 6 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
.PHONY: help up down restart training-up training-down monitoring-up monitoring-down \
up-all down-all lint test build-images pull-images smoke ci-env
up-all down-all lint test build-images pull-images smoke ci-env gateway-auth

# Product Python paths linted in CI (Phase 1A). Expand later if needed.
LINT_PATHS := src services scripts
Expand Down Expand Up @@ -37,7 +37,8 @@ help:
@echo " make build-images - Build all product Docker images"
@echo " make pull-images - Pull product images from GHCR (GHCR_TAG=main|sha-...)"
@echo " make smoke - Run Compose smoke scripts (stack must be up)"
@echo " make ci-env - Copy .env.example -> .env for local/CI Compose"
@echo " make ci-env - Copy .env.example -> .env if missing"
@echo " make gateway-auth - Write nginx/.htpasswd-* from GATEWAY_* in .env"

up:
docker compose up -d --build
Expand Down Expand Up @@ -78,6 +79,9 @@ ci-env:
echo "Created .env from .env.example"; \
fi

gateway-auth: ci-env
./scripts/prepare_gateway_auth.sh

lint:
ruff check $(LINT_PATHS)

Expand Down
7 changes: 4 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# CAFA-5 MLOps Solution
# ProSeqGO: Protein Sequence Gene Onthology prediction

End-to-end MLOps platform for CAFA-5 protein function prediction (sequence -> embedding -> GO terms), with model lifecycle management, secured gateway routing, and production-oriented monitoring.
End-to-end MLOps platform for protein function prediction (sequence -> embedding -> GO terms), with model lifecycle management, secured gateway routing, and production-oriented monitoring.

## Problem This Project Solves

Expand Down Expand Up @@ -469,6 +469,7 @@ make build-images # Build all five product images
make pull-images # Pull product images from GHCR (GHCR_TAG=main|sha-...)
make smoke # Smoke scripts (Compose stack must already be up)
make ci-env # Copy .env.example → .env if missing
make gateway-auth # Write nginx/.htpasswd-* from GATEWAY_* in .env
```

## CI (GitHub Actions)
Expand All @@ -477,7 +478,7 @@ PR and `main` pushes run **lint**, **unit tests**, and **parallel image builds**

- Registry: **GHCR** (`ghcr.io/behroooz/proseqgo-*`)
- CI does **not** run training/GPU/retrain jobs
- Compose in CI will use `make ci-env` (`.env.example` only)—never commit real secrets
- Compose secrets: `.env` from `.env.example`; gateway Basic Auth via `make gateway-auth` (`GATEWAY_ADMIN_*` ≠ `GATEWAY_USER_*`)
- Local image rebuild: `make build-images`; pull published: `make pull-images`

## Service-Specific Documentation
Expand Down
3 changes: 3 additions & 0 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -315,6 +315,9 @@ services:
networks: [proseqgo]
environment:
GATEWAY_BASE_URL: http://nginx
# Predict-route credentials (public user). Must match make gateway-auth / .htpasswd-user.
GATEWAY_USER: ${GATEWAY_USER:-user}
GATEWAY_USER_PASSWORD: ${GATEWAY_USER_PASSWORD:-change-me-gateway-user}
depends_on:
- embedding-api
- go-prediction-api
Expand Down
1 change: 1 addition & 0 deletions nginx/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ This milestone introduced NGINX as the single ingress gateway for the CAFA-5 MLO
- **Authentication segmentation:**
- Admin-only access for embedding endpoints and MLflow (`.htpasswd-admin`).
- User-level access for prediction endpoints (`.htpasswd-user`).
- Local discipline: set distinct `GATEWAY_ADMIN_*` and `GATEWAY_USER_*` in `.env`, then `make gateway-auth`.
- **Route-level rate limiting:** Separate request budgets for admin and prediction paths with `429` on limit exceed.
- **Route-specific payload limits:** Enforced `client_max_body_size` per endpoint group to protect upstream services.
- **Hardened proxy timeouts:** Long read/send timeouts for model/training workloads; bounded connect timeout.
Expand Down
36 changes: 36 additions & 0 deletions scripts/load_gateway_env.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
#!/usr/bin/env bash
# Load GATEWAY_* from repo .env into the current shell without executing arbitrary lines.
#
# Usage:
# source scripts/load_gateway_env.sh
# load_gateway_env /path/to/repo/root

load_gateway_env() {
local repo_root="${1:-.}"
local env_file="${repo_root}/.env"
local key val line

if [[ -f "${env_file}" ]]; then
while IFS= read -r line || [[ -n "${line}" ]]; do
[[ -z "${line}" || "${line}" =~ ^[[:space:]]*# ]] && continue
case "${line}" in
GATEWAY_*=*)
key="${line%%=*}"
val="${line#*=}"
if [[ "${val}" =~ ^\".*\"$ || "${val}" =~ ^\'.*\'$ ]]; then
val="${val:1:-1}"
fi
if [[ -z "${!key:-}" ]]; then
printf -v "${key}" '%s' "${val}"
export "${key?}"
fi
;;
esac
done <"${env_file}"
fi

export GATEWAY_ADMIN_USER="${GATEWAY_ADMIN_USER:-admin}"
export GATEWAY_ADMIN_PASSWORD="${GATEWAY_ADMIN_PASSWORD:-change-me-gateway-admin}"
export GATEWAY_USER="${GATEWAY_USER:-user}"
export GATEWAY_USER_PASSWORD="${GATEWAY_USER_PASSWORD:-change-me-gateway-user}"
}
49 changes: 49 additions & 0 deletions scripts/prepare_gateway_auth.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
#!/usr/bin/env bash
# Generate nginx Basic Auth files from .env (GATEWAY_*).
# Files are gitignored (nginx/.htpasswd-*). Never commit real passwords.
#
# GATEWAY_ADMIN_USER / GATEWAY_ADMIN_PASSWORD → .htpasswd-admin
# ( /api/v1/* admin routes, /mlflow, /api/train )
# GATEWAY_USER / GATEWAY_USER_PASSWORD → .htpasswd-user
# ( /api/predict/*, predict-go-* )
#
# Admin and public-user passwords must differ.

set -euo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
# shellcheck source=load_gateway_env.sh
source "${REPO_ROOT}/scripts/load_gateway_env.sh"
load_gateway_env "${REPO_ROOT}"

NGINX_DIR="${REPO_ROOT}/nginx"

if ! command -v htpasswd >/dev/null 2>&1; then
echo "htpasswd not found. Install apache2-utils (Debian/Ubuntu) or httpd-tools (RHEL)." >&2
exit 1
fi

if [[ -z "${GATEWAY_ADMIN_PASSWORD}" || -z "${GATEWAY_USER_PASSWORD}" ]]; then
echo "GATEWAY_ADMIN_PASSWORD and GATEWAY_USER_PASSWORD must be set (see .env.example)." >&2
exit 1
fi

if [[ "${GATEWAY_ADMIN_PASSWORD}" == "${GATEWAY_USER_PASSWORD}" ]]; then
echo "Refusing to write htpasswd: GATEWAY_ADMIN_PASSWORD must differ from GATEWAY_USER_PASSWORD." >&2
exit 1
fi

if [[ "${GATEWAY_ADMIN_USER}" == "${GATEWAY_USER}" ]]; then
echo "Refusing to write htpasswd: GATEWAY_ADMIN_USER must differ from GATEWAY_USER." >&2
exit 1
fi

mkdir -p "${NGINX_DIR}"
htpasswd -nbB "${GATEWAY_ADMIN_USER}" "${GATEWAY_ADMIN_PASSWORD}" >"${NGINX_DIR}/.htpasswd-admin"
htpasswd -nbB "${GATEWAY_USER}" "${GATEWAY_USER_PASSWORD}" >"${NGINX_DIR}/.htpasswd-user"

echo "Wrote ${NGINX_DIR}/.htpasswd-admin and ${NGINX_DIR}/.htpasswd-user"
echo "Admin (ops): ${GATEWAY_ADMIN_USER}"
echo "User (predict): ${GATEWAY_USER}"
echo "Next: recreate streamlit if needed → docker compose up -d streamlit-ui"
echo "UI should use the predict user; admin password is for /mlflow and admin APIs."
5 changes: 3 additions & 2 deletions services/streamlit-ui/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ The Streamlit app in `services/streamlit-ui/app.py` includes:
- protein sequence input,
- `top_k` input (`1..500`),
- gateway base URL input (defaults from `GATEWAY_BASE_URL` env var),
- API username/password fields,
- API username/password fields (defaults from `GATEWAY_USER` / `GATEWAY_USER_PASSWORD`),
- optional TLS verification toggle.
- Input QC and validation:
- trims whitespace/newlines from sequence,
Expand Down Expand Up @@ -100,7 +100,8 @@ Important auth boundary:

- Streamlit UI path itself is public (`/ui/`).
- Prediction endpoint `/api/v1/predict-go-from-sequences` still uses NGINX basic auth (`.htpasswd-user`).
- Credentials are provided by user in the Streamlit form and sent with each API request.
- Defaults come from `.env` via Compose (`GATEWAY_USER` / `GATEWAY_USER_PASSWORD`). Run `make gateway-auth` so htpasswd matches.
- Admin credentials (`GATEWAY_ADMIN_*`) are for `/mlflow` and admin `/api/v1` routes — not the public UI defaults.

## Run And Access

Expand Down
7 changes: 5 additions & 2 deletions services/streamlit-ui/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@
"prediction APIs through the NGINX gateway."
)
DEFAULT_GATEWAY_URL = os.getenv("GATEWAY_BASE_URL", "http://localhost")
# Public predict-route defaults (must match nginx/.htpasswd-user from make gateway-auth).
DEFAULT_API_USERNAME = os.getenv("GATEWAY_USER", "")
DEFAULT_API_PASSWORD = os.getenv("GATEWAY_USER_PASSWORD", "")
PREDICT_SEQUENCES_ENDPOINT = "/api/v1/predict-go-from-sequences"
PREDICT_FASTA_ENDPOINT = "/api/v1/predict-go-from-fasta"
MAX_TOP_K = 500
Expand Down Expand Up @@ -184,8 +187,8 @@ def _render_shared_connection_fields(
) -> tuple[str, str, str, bool, int]:
top_k = st.number_input("top_k", min_value=1, max_value=MAX_TOP_K, value=10, step=1)
gateway_base_url_input = st.text_input("Gateway base URL", value=gateway_base_url)
username = st.text_input("API username")
password = st.text_input("API password", type="password")
username = st.text_input("API username", value=DEFAULT_API_USERNAME)
password = st.text_input("API password", type="password", value=DEFAULT_API_PASSWORD)
verify_tls = st.checkbox("Verify TLS", value=verify_tls_default)
return gateway_base_url_input, username, password, verify_tls, int(top_k)

Expand Down
9 changes: 8 additions & 1 deletion tests/smoke/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,14 @@
#
# These are not unit tests: they expect docker compose services to be up.
#
# From repo root:
# Gateway credentials live in `.env` (`GATEWAY_ADMIN_*`, `GATEWAY_USER_*`).
# Sync nginx htpasswd (admin ≠ public user):
#
# make gateway-auth
#
# From repo root (stack must be up):
# ./tests/smoke/smoke_embedding_api.sh
# ./tests/smoke/test_embedding_worker_crash_recovery.sh
# MLFLOW_TRACKING_URI=http://127.0.0.1/mlflow python tests/smoke/mlflow_smoke_test.py
#
# Admin credentials are used for /api/v1/jobs*; public user for predict-go-*.
46 changes: 29 additions & 17 deletions tests/smoke/smoke_embedding_api.sh
Original file line number Diff line number Diff line change
@@ -1,33 +1,45 @@
#!/usr/bin/env bash
# Smoke test for the Embedding API (direct uvicorn or via nginx gateway on port 80).
# Usage (from repo root):
# make gateway-auth # once: sync nginx htpasswd from .env
# ./tests/smoke/smoke_embedding_api.sh
# BASE_URL=http://127.0.0.1:8000 ./tests/smoke/smoke_embedding_api.sh
# CURL_INSECURE=1 API_USER=user API_PASS=secret \
# BASE_URL=https://localhost ./tests/smoke/smoke_embedding_api.sh
#
# Credentials come from .env:
# GATEWAY_ADMIN_* → /api/v1/health, /api/v1/jobs, artifacts
# GATEWAY_USER_* → /api/v1/predict-go-from-fasta
# Overrides: ADMIN_USER/ADMIN_PASS, API_USER/API_PASS (predict user).

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
# shellcheck source=../../scripts/load_gateway_env.sh
source "${REPO_ROOT}/scripts/load_gateway_env.sh"
load_gateway_env "${REPO_ROOT}"

BASE_URL="${BASE_URL:-http://127.0.0.1}"
FASTA_EXAMPLE="${REPO_ROOT}/examples/small_sequences.fasta"
MAX_FASTA_UPLOAD_BYTES=$((5 * 1024 * 1024))

CURL_OPTS=(-sS)
ADMIN_USER="${ADMIN_USER:-${GATEWAY_ADMIN_USER}}"
ADMIN_PASS="${ADMIN_PASS:-${GATEWAY_ADMIN_PASSWORD}}"
API_USER="${API_USER:-${GATEWAY_USER}}"
API_PASS="${API_PASS:-${GATEWAY_USER_PASSWORD}}"

CURL_BASE=(-sS)
if [[ -n "${CURL_INSECURE:-}" ]]; then
CURL_OPTS+=(-k)
fi
if [[ -n "${API_USER:-}" && -n "${API_PASS:-}" ]]; then
CURL_OPTS+=(-u "${API_USER}:${API_PASS}")
CURL_BASE+=(-k)
fi
ADMIN_CURL=("${CURL_BASE[@]}" -u "${ADMIN_USER}:${ADMIN_PASS}")
USER_CURL=("${CURL_BASE[@]}" -u "${API_USER}:${API_PASS}")

echo "==> Health: GET ${BASE_URL}/api/v1/health"
curl "${CURL_OPTS[@]}" "${BASE_URL}/api/v1/health"
echo "==> Health: GET ${BASE_URL}/api/v1/health (admin)"
curl "${ADMIN_CURL[@]}" "${BASE_URL}/api/v1/health"
echo

echo "==> Submit job: POST ${BASE_URL}/api/v1/jobs"
RESP="$(curl "${CURL_OPTS[@]}" -X POST "${BASE_URL}/api/v1/jobs" \
echo "==> Submit job: POST ${BASE_URL}/api/v1/jobs (admin)"
RESP="$(curl "${ADMIN_CURL[@]}" -X POST "${BASE_URL}/api/v1/jobs" \
-H "Content-Type: application/json" \
-d '{
"stage": "test",
Expand All @@ -47,7 +59,7 @@ echo "==> Job ID: ${JOB_ID}"

echo "==> Poll until succeeded (max ~120s)"
for _ in $(seq 1 60); do
ST="$(curl "${CURL_OPTS[@]}" "${BASE_URL}/api/v1/jobs/${JOB_ID}")"
ST="$(curl "${ADMIN_CURL[@]}" "${BASE_URL}/api/v1/jobs/${JOB_ID}")"
STATUS="$(printf '%s' "$ST" | python3 -c "import sys, json; print(json.load(sys.stdin)['status'])")"
if [[ "$STATUS" == "succeeded" ]]; then
echo "$ST" | python3 -m json.tool
Expand All @@ -67,9 +79,9 @@ fi

OUT_DIR="$(mktemp -d)"
echo "==> Download artifacts to ${OUT_DIR}"
curl "${CURL_OPTS[@]}" -o "${OUT_DIR}/test_ids.npy" \
curl "${ADMIN_CURL[@]}" -o "${OUT_DIR}/test_ids.npy" \
"${BASE_URL}/api/v1/jobs/${JOB_ID}/artifacts/test_ids.npy"
curl "${CURL_OPTS[@]}" -o "${OUT_DIR}/test_embeddings.npy" \
curl "${ADMIN_CURL[@]}" -o "${OUT_DIR}/test_embeddings.npy" \
"${BASE_URL}/api/v1/jobs/${JOB_ID}/artifacts/test_embeddings.npy"

echo "==> Verify shapes (expect N=2, D=1280 for esm2, float32)"
Expand All @@ -87,8 +99,8 @@ assert str(emb.dtype) == "float32"
print("OK")
PY

echo "==> Predict GO from FASTA: POST ${BASE_URL}/api/v1/predict-go-from-fasta"
PRED_RESP="$(curl "${CURL_OPTS[@]}" --max-time 1800 -X POST \
echo "==> Predict GO from FASTA: POST ${BASE_URL}/api/v1/predict-go-from-fasta (user)"
PRED_RESP="$(curl "${USER_CURL[@]}" --max-time 1800 -X POST \
"${BASE_URL}/api/v1/predict-go-from-fasta" \
-F "fasta_file=@${FASTA_EXAMPLE}" \
-F "backend=esm2" \
Expand Down Expand Up @@ -116,7 +128,7 @@ PY
echo "==> FASTA upload too large: expect HTTP 413 (max ${MAX_FASTA_UPLOAD_BYTES} bytes)"
LARGE_FASTA="$(mktemp)"
python3 -c "import sys; sys.stdout.buffer.write(b'x' * (${MAX_FASTA_UPLOAD_BYTES} + 1))" >"${LARGE_FASTA}"
HTTP_CODE="$(curl "${CURL_OPTS[@]}" -o /dev/null -w "%{http_code}" -X POST \
HTTP_CODE="$(curl "${USER_CURL[@]}" -o /dev/null -w "%{http_code}" -X POST \
"${BASE_URL}/api/v1/predict-go-from-fasta" \
-F "fasta_file=@${LARGE_FASTA};type=text/plain" \
-F "backend=esm2")"
Expand Down
Loading