ci: add lint, tests, image builds, and GHCR publish

.dockerignore

@@ -0,0 +1,27 @@
+# Keep Compose/CI build context small. Data and secrets are mounted at runtime.
+.git
+.gitignore
+.github
+.venv
+venv
+__pycache__/
+*.pyc
+*.pyo
+*.npy
+*.pt
+*.pth
+*.safetensors
+data/
+outputs/
+backups/
+archives/
+notes/
+notebooks/
+docs/
+tests/
+.cursor/
+.idea/
+.vscode/
+.env
+.env.*
+!.env.example

.github/CI.md

@@ -0,0 +1,50 @@
+# Continuous Integration notes (ProSeqGo)
+
+## What runs today
+
+| Trigger | Job | Command |
+|---------|-----|---------|
+| `pull_request`, `push` to `main` | Lint | `make lint` → `ruff check src services scripts` |
+| `pull_request`, `push` to `main` | Unit tests (after lint) | `make test` → `pytest tests/unit -q` |
+| `pull_request` | Image builds (after lint, parallel) | Buildx build of all 5 product images (**no push**) |
+| `push` to `main` | Image builds + GHCR publish | Build and push `sha-<fullsha>` + `main` tags |
+
+Unit tests are **Docker/GPU/network free**. They cover sequence normalization, config loading, API schema bounds, embedding vector validation, and UI input gates.
+
+### Image builds
+
+| Image | Dockerfile | GHCR repository |
+|-------|------------|-----------------|
+| `proseqgo-embedding-api` | `docker/docker_embedding/Dockerfile.embedding-api` | `ghcr.io/behroooz/proseqgo-embedding-api` |
+| `proseqgo-go-prediction-api` | `docker/docker_go_term/Dockerfile.api` | `ghcr.io/behroooz/proseqgo-go-prediction-api` |
+| `proseqgo-streamlit-ui` | `docker/docker_streamlit/Dockerfile.streamlit` | `ghcr.io/behroooz/proseqgo-streamlit-ui` |
+| `proseqgo-trainer-api` | `docker/docker_training/Dockerfile.training` | `ghcr.io/behroooz/proseqgo-trainer-api` |
+| `proseqgo-mlflow` | `docker/docker_mlflow/Dockerfile` | `ghcr.io/behroooz/proseqgo-mlflow` |
+
+Policy:
+
+```text
+PR     → build only (GHA layer cache, no push)
+main   → build + push sha-<fullsha> and moving main
+```
+
+- Torch images use **CPU wheels** in CI (`TORCH_INDEX_URL=.../cpu`) for smaller/faster builds; local Compose still defaults to CUDA index
+- Local build: `make build-images`
+- Pull published images: `make pull-images` or `GHCR_TAG=sha-<fullsha> make pull-images`
+- First publish happens after this workflow runs on **`main`**. Packages may be private by default; set package visibility in GitHub Packages if others need to pull.
+
+## Compose / secrets in CI (Phase 3)
+
+Workflows must never commit real secrets. Pattern:
+
+```bash
+make ci-env   # copies .env.example → .env if missing
+```
+
+Use throwaway passwords from `.env.example` only inside ephemeral CI runners.
+
+## Explicit non-goals for CI
+
+- No training / GPU / full retrain jobs in PR or `main` CI
+- No Training API profile in automated smoke (serving stack only)
+- Compose smoke lands in Phase 3

.github/workflows/ci.yml

@@ -0,0 +1,151 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+concurrency:
+  group: ci-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  actions: write
+  packages: write
+
+env:
+  # GHCR requires lowercase owner/image names.
+  REGISTRY: ghcr.io
+
+jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      # Lint-only install: do not pip install the project (would pull torch/HF).
+      - name: Install Ruff
+        run: pip install "ruff>=0.1"
+
+      - name: Ruff check
+        run: make lint
+
+  test:
+    name: Unit tests
+    runs-on: ubuntu-latest
+    needs: lint
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      # Slim install: unit tests must not need torch/transformers/streamlit.
+      - name: Install test dependencies
+        run: pip install "pytest>=7.0" "pydantic>=2.0" "pyyaml>=6.0" "numpy>=1.24"
+
+      - name: Run unit tests
+        run: make test
+
+  # Phase 1C/2: build all product images; push to GHCR only on main.
+  build-images:
+    name: Build ${{ matrix.name }}
+    runs-on: ubuntu-latest
+    needs: lint
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: embedding-api
+            dockerfile: docker/docker_embedding/Dockerfile.embedding-api
+            image: proseqgo-embedding-api
+            build_args: TORCH_INDEX_URL=https://download.pytorch.org/whl/cpu
+          - name: go-prediction-api
+            dockerfile: docker/docker_go_term/Dockerfile.api
+            image: proseqgo-go-prediction-api
+            build_args: TORCH_INDEX_URL=https://download.pytorch.org/whl/cpu
+          - name: streamlit-ui
+            dockerfile: docker/docker_streamlit/Dockerfile.streamlit
+            image: proseqgo-streamlit-ui
+            build_args: ""
+          - name: trainer-api
+            dockerfile: docker/docker_training/Dockerfile.training
+            image: proseqgo-trainer-api
+            build_args: TORCH_INDEX_URL=https://download.pytorch.org/whl/cpu
+          - name: mlflow
+            dockerfile: docker/docker_mlflow/Dockerfile
+            image: proseqgo-mlflow
+            build_args: ""
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Free disk space
+        run: |
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \
+            /usr/local/share/powershell /usr/share/swift /usr/local/graalvm \
+            /usr/local/.ghcup || true
+          df -h
+
+      - name: Set image name and push flag
+        id: meta
+        run: |
+          owner="$(echo '${{ github.repository_owner }}' | tr '[:upper:]' '[:lower:]')"
+          echo "image=${{ env.REGISTRY }}/${owner}/${{ matrix.image }}" >> "$GITHUB_OUTPUT"
+          if [ '${{ github.event_name }}' = 'push' ] && [ '${{ github.ref }}' = 'refs/heads/main' ]; then
+            echo "push=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "push=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GHCR
+        if: steps.meta.outputs.push == 'true'
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # PR / non-main: prove the image builds; do not push.
+      - name: Build image (no push)
+        if: steps.meta.outputs.push != 'true'
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ${{ matrix.dockerfile }}
+          push: false
+          outputs: type=cacheonly
+          tags: ${{ steps.meta.outputs.image }}:sha-${{ github.sha }}
+          build-args: ${{ matrix.build_args }}
+          cache-from: type=gha,scope=build-${{ matrix.name }}
+          cache-to: type=gha,mode=max,scope=build-${{ matrix.name }}
+
+      # main: publish immutable sha tag + moving main tag.
+      - name: Build and push image
+        if: steps.meta.outputs.push == 'true'
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ${{ matrix.dockerfile }}
+          push: true
+          tags: |
+            ${{ steps.meta.outputs.image }}:sha-${{ github.sha }}
+            ${{ steps.meta.outputs.image }}:main
+          build-args: ${{ matrix.build_args }}
+          cache-from: type=gha,scope=build-${{ matrix.name }}
+          cache-to: type=gha,mode=max,scope=build-${{ matrix.name }}
+          provenance: false

Makefile

@@ -1,4 +1,28 @@
-.PHONY: help up down training-up training-down monitoring-up monitoring-down
+.PHONY: help up down restart training-up training-down monitoring-up monitoring-down \
+	up-all down-all lint test build-images pull-images smoke ci-env
+
+# Product Python paths linted in CI (Phase 1A). Expand later if needed.
+LINT_PATHS := src services scripts
+PYTHON ?= python3
+
+# Image names match docker-compose.yml local tags.
+EMBEDDING_IMAGE ?= proseqgo-embedding-api:local
+GO_PRED_IMAGE ?= proseqgo-go-prediction-api:local
+STREAMLIT_IMAGE ?= proseqgo-streamlit-ui:local
+TRAINER_IMAGE ?= proseqgo-trainer-api:local
+MLFLOW_IMAGE ?= proseqgo-mlflow:local
+# Local default matches GPU Compose; CI passes cpu index via TORCH_INDEX_URL.
+TORCH_INDEX_URL ?= https://download.pytorch.org/whl/cu132
+
+# GHCR (Phase 2). Owner must be lowercase. Tag: main | sha-<fullsha>
+GHCR_OWNER ?= behroooz
+GHCR_REGISTRY ?= ghcr.io
+GHCR_TAG ?= main
+GHCR_EMBEDDING_IMAGE ?= $(GHCR_REGISTRY)/$(GHCR_OWNER)/proseqgo-embedding-api:$(GHCR_TAG)
+GHCR_GO_PRED_IMAGE ?= $(GHCR_REGISTRY)/$(GHCR_OWNER)/proseqgo-go-prediction-api:$(GHCR_TAG)
+GHCR_STREAMLIT_IMAGE ?= $(GHCR_REGISTRY)/$(GHCR_OWNER)/proseqgo-streamlit-ui:$(GHCR_TAG)
+GHCR_TRAINER_IMAGE ?= $(GHCR_REGISTRY)/$(GHCR_OWNER)/proseqgo-trainer-api:$(GHCR_TAG)
+GHCR_MLFLOW_IMAGE ?= $(GHCR_REGISTRY)/$(GHCR_OWNER)/proseqgo-mlflow:$(GHCR_TAG)
 
 help:
 	@echo "Available targets:"
@@ -8,6 +32,12 @@ help:
 	@echo "  make training-down    - Stop services started with the training profile"
 	@echo "  make monitoring-up    - Start services with the monitoring profile"
 	@echo "  make monitoring-down  - Stop services started with the monitoring profile"
+	@echo "  make lint             - Ruff check on src/ services/ scripts/"
+	@echo "  make test             - Unit tests (tests/unit; Phase 1B)"
+	@echo "  make build-images     - Build all product Docker images"
+	@echo "  make pull-images      - Pull product images from GHCR (GHCR_TAG=main|sha-...)"
+	@echo "  make smoke            - Run Compose smoke scripts (stack must be up)"
+	@echo "  make ci-env           - Copy .env.example -> .env for local/CI Compose"
 
 up:
 	docker compose up -d --build
@@ -26,13 +56,59 @@ training-down:
 	docker compose --profile training down
 
 monitoring-up:
-	docker compose --profile monitoring up -d
+	docker compose --profile monitoring up -d --build
 
 monitoring-down:
 	docker compose --profile monitoring down
 
 up-all:
-	docker compose --profile monitoring up -d --build
+	docker compose --profile monitoring --profile training up -d --build
 
 down-all:
-	docker compose down
+	docker compose --profile monitoring --profile training down
+
+# --- CI / quality (same commands locally and in GitHub Actions) ---
+
+ci-env:
+	@test -f .env.example || (echo "Missing .env.example"; exit 1)
+	@if [ -f .env ]; then \
+		echo ".env already present (left unchanged)"; \
+	else \
+		cp .env.example .env; \
+		echo "Created .env from .env.example"; \
+	fi
+
+lint:
+	ruff check $(LINT_PATHS)
+
+test:
+	$(PYTHON) -m pytest tests/unit -q
+
+build-images:
+	docker build \
+		--build-arg TORCH_INDEX_URL=$(TORCH_INDEX_URL) \
+		-f docker/docker_embedding/Dockerfile.embedding-api \
+		-t $(EMBEDDING_IMAGE) .
+	docker build \
+		--build-arg TORCH_INDEX_URL=$(TORCH_INDEX_URL) \
+		-f docker/docker_go_term/Dockerfile.api \
+		-t $(GO_PRED_IMAGE) .
+	docker build -f docker/docker_streamlit/Dockerfile.streamlit -t $(STREAMLIT_IMAGE) .
+	docker build \
+		--build-arg TORCH_INDEX_URL=$(TORCH_INDEX_URL) \
+		-f docker/docker_training/Dockerfile.training \
+		-t $(TRAINER_IMAGE) .
+	docker build -f docker/docker_mlflow/Dockerfile -t $(MLFLOW_IMAGE) .
+
+pull-images:
+	docker pull $(GHCR_EMBEDDING_IMAGE)
+	docker pull $(GHCR_GO_PRED_IMAGE)
+	docker pull $(GHCR_STREAMLIT_IMAGE)
+	docker pull $(GHCR_TRAINER_IMAGE)
+	docker pull $(GHCR_MLFLOW_IMAGE)
+
+# Expects default Compose stack already healthy. Does not start training/GPU jobs.
+smoke:
+	./tests/smoke/smoke_embedding_api.sh
+	@echo "Optional: MLFLOW_TRACKING_URI=http://127.0.0.1/mlflow python tests/smoke/mlflow_smoke_test.py"
+	@echo "Optional (heavier): ./tests/smoke/test_embedding_worker_crash_recovery.sh"

README.md

@@ -58,7 +58,11 @@ CAFA-5-MLOps-solution/
 ├── outputs/                      # Splits, labels, checkpoints, artifacts, submissions
 ├── scripts/                      # CLI pipeline entrypoints (preprocess/train/evaluate/predict)
 ├── tests/
+│   ├── unit/                     # Fast pytest suite (no Docker/GPU; CI Phase 1B)
 │   └── smoke/                    # Compose smoke/acceptance checks (not unit tests)
+├── .github/
+│   ├── workflows/ci.yml          # PR/main CI (lint today; tests/images later)
+│   └── CI.md                     # CI scope, GHCR plan, non-goals
 ├── services/
 │   ├── embedding-api/            # Async embedding jobs + sequence->GO orchestration endpoint
 │   ├── go-prediction-api/        # Embedding->GO inference API
@@ -459,8 +463,23 @@ make training-up
 make training-down
 make monitoring-up
 make monitoring-down
+make lint            # Ruff on src/ services/ scripts/
+make test            # Unit tests (tests/unit)
+make build-images    # Build all five product images
+make pull-images     # Pull product images from GHCR (GHCR_TAG=main|sha-...)
+make smoke           # Smoke scripts (Compose stack must already be up)
+make ci-env          # Copy .env.example → .env if missing
 ```
 
+## CI (GitHub Actions)
+
+PR and `main` pushes run **lint**, **unit tests**, and **parallel image builds**. Merges to `main` also **publish to GHCR** (`sha-<fullsha>` + `main`). See [`.github/CI.md`](.github/CI.md).
+
+- Registry: **GHCR** (`ghcr.io/behroooz/proseqgo-*`)
+- CI does **not** run training/GPU/retrain jobs
+- Compose in CI will use `make ci-env` (`.env.example` only)—never commit real secrets
+- Local image rebuild: `make build-images`; pull published: `make pull-images`
+
 ## Service-Specific Documentation
 
 - `services/embedding-api/README.md`