From 25f14c2f8c07fe16a9035c4a78900179dc0fb13a Mon Sep 17 00:00:00 2001 From: Behrouz Mirabdi Date: Fri, 24 Jul 2026 16:38:06 +0200 Subject: [PATCH 1/9] refactor(netrwork): change network name from cafa5 to proseqgo; also change the image names contained cafa5 --- docker-compose.yml | 50 +++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 00f5fa7..a3d8c35 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,7 @@ # Monitoring: docker compose --profile monitoring up # Copy .env.example to .env and set secrets before first run. networks: - cafa5: + proseqgo: driver: bridge volumes: @@ -17,7 +17,7 @@ volumes: services: postgres: image: postgres:16-alpine - networks: [cafa5] + networks: [proseqgo] environment: POSTGRES_USER: ${POSTGRES_USER} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} @@ -35,7 +35,7 @@ services: redis: image: redis:7-alpine - networks: [cafa5] + networks: [proseqgo] command: ["redis-server", "--appendonly", "yes"] volumes: - redis_data:/data @@ -49,7 +49,7 @@ services: minio: image: minio/minio:latest - networks: [cafa5] + networks: [proseqgo] command: server /data --console-address ":9001" environment: MINIO_ROOT_USER: ${MINIO_ROOT_USER} @@ -69,7 +69,7 @@ services: minio-init: image: minio/mc:latest - networks: [cafa5] + networks: [proseqgo] depends_on: minio: condition: service_healthy @@ -89,8 +89,8 @@ services: build: context: . dockerfile: docker/docker_mlflow/Dockerfile - image: cafa5-mlflow:local - networks: [cafa5] + image: proseqgo-mlflow:local + networks: [proseqgo] depends_on: postgres: condition: service_healthy @@ -113,7 +113,7 @@ services: postgres-backup: image: prodrigestivill/postgres-backup-local:16 - networks: [cafa5] + networks: [proseqgo] depends_on: postgres: condition: service_healthy @@ -133,7 +133,7 @@ services: backup-offload: image: minio/mc:latest - networks: [cafa5] + networks: [proseqgo] depends_on: minio: condition: service_healthy @@ -157,10 +157,10 @@ services: build: context: . dockerfile: docker/docker_training/Dockerfile.training - image: cafa5-trainer-api:local + image: proseqgo-trainer-api:local working_dir: /app profiles: ["training"] - networks: [cafa5] + networks: [proseqgo] environment: &trainer-env PYTHONUNBUFFERED: "1" PYTHONPATH: /app:/app/services/training-api @@ -193,10 +193,10 @@ services: build: context: . dockerfile: docker/docker_training/Dockerfile.training - image: cafa5-trainer-api:local + image: proseqgo-trainer-api:local working_dir: /app/services/training-api profiles: ["training"] - networks: [cafa5] + networks: [proseqgo] gpus: all stop_grace_period: 2h environment: @@ -220,9 +220,9 @@ services: build: context: . dockerfile: docker/docker_embedding/Dockerfile.embedding-api - image: cafa5-embedding-api:local + image: proseqgo-embedding-api:local working_dir: /app/services/embedding-api - networks: [cafa5] + networks: [proseqgo] environment: &embedding-env PYTHONUNBUFFERED: "1" PYTHONPATH: /app:/app/services/embedding-api @@ -255,9 +255,9 @@ services: build: context: . dockerfile: docker/docker_embedding/Dockerfile.embedding-api - image: cafa5-embedding-api:local + image: proseqgo-embedding-api:local working_dir: /app/services/embedding-api - networks: [cafa5] + networks: [proseqgo] gpus: all stop_grace_period: 3700s environment: @@ -283,9 +283,9 @@ services: build: context: . dockerfile: docker/docker_go_term/Dockerfile.api - image: cafa5-go-prediction-api:local + image: proseqgo-go-prediction-api:local working_dir: /app/services/go-prediction-api - networks: [cafa5] + networks: [proseqgo] gpus: all environment: PYTHONUNBUFFERED: "1" @@ -310,9 +310,9 @@ services: build: context: . dockerfile: docker/docker_streamlit/Dockerfile.streamlit - image: cafa5-streamlit-ui:local + image: proseqgo-streamlit-ui:local working_dir: /app/services/streamlit-ui - networks: [cafa5] + networks: [proseqgo] environment: GATEWAY_BASE_URL: http://nginx depends_on: @@ -322,7 +322,7 @@ services: nginx: image: nginx:latest - networks: [cafa5] + networks: [proseqgo] ports: - "80:80" volumes: @@ -339,7 +339,7 @@ services: redis-exporter: image: oliver006/redis_exporter:v1.66.0 profiles: ["monitoring"] - networks: [cafa5] + networks: [proseqgo] environment: REDIS_ADDR: redis://redis:6379 depends_on: @@ -351,7 +351,7 @@ services: image: prom/prometheus:latest container_name: prometheus profiles: ["monitoring"] - networks: [cafa5] + networks: [proseqgo] command: - --config.file=/etc/prometheus/prometheus.yml - --storage.tsdb.retention.time=15d @@ -372,7 +372,7 @@ services: image: grafana/grafana:latest container_name: grafana profiles: ["monitoring"] - networks: [cafa5] + networks: [proseqgo] environment: GF_SECURITY_ADMIN_USER: admin GF_SECURITY_ADMIN_PASSWORD: admin From dfd76d8e770a6a011c4f7462ecdb80c80bafb0a7 Mon Sep 17 00:00:00 2001 From: Behrouz Mirabdi Date: Fri, 24 Jul 2026 16:40:36 +0200 Subject: [PATCH 2/9] docs: change the name to ProSeqGO --- monitoring/prometheus.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitoring/prometheus.yml b/monitoring/prometheus.yml index fd2ffc1..253b9b1 100644 --- a/monitoring/prometheus.yml +++ b/monitoring/prometheus.yml @@ -1,4 +1,4 @@ -# Prometheus configuration for CAFA-5 MLOps solution +# Prometheus configuration for ProSeqGO global: scrape_interval: 30s evaluation_interval: 30s From b1542adeb1599e107912c9d90a7932a7703428e8 Mon Sep 17 00:00:00 2001 From: Behrouz Mirabdi Date: Fri, 24 Jul 2026 16:44:25 +0200 Subject: [PATCH 3/9] refactor(monitoring): change metric name strings --- services/embedding-api/main.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/services/embedding-api/main.py b/services/embedding-api/main.py index b3bdd25..7f949db 100644 --- a/services/embedding-api/main.py +++ b/services/embedding-api/main.py @@ -40,44 +40,44 @@ SERVICE_NAME = "embedding-api" HTTP_REQUESTS_TOTAL = Counter( - "cafa5_http_requests_total", + "http_requests_total", "Total number of HTTP requests.", labelnames=("service", "route", "method", "status_code"), registry=registry, ) HTTP_REQUEST_DURATION_SECONDS = Histogram( - "cafa5_http_request_duration_seconds", + "http_request_duration_seconds", "HTTP request duration in seconds.", labelnames=("service", "route", "method", "status_code"), registry=registry, ) HTTP_IN_FLIGHT_REQUESTS = Gauge( - "cafa5_http_in_flight_requests", + "http_in_flight_requests", "Number of in-flight HTTP requests.", labelnames=("service",), registry=registry, ) EMBEDDING_SEQUENCE_LENGTH = Histogram( - "cafa5_embedding_sequence_length", + "embedding_sequence_length", "Observed amino-acid sequence lengths partitioned by embedding backend.", labelnames=("backend",), buckets=(16, 32, 64, 128, 256, 512, 1024, 1280, 2048, 4096, 8192, float("inf")), registry=registry, ) EMBEDDING_DIMENSION_MISMATCHES_TOTAL = Counter( - "cafa5_embedding_dimension_mismatch_total", + "embedding_dimension_mismatch_total", "Total number of embedding dimension mismatches detected before GO inference.", registry=registry, ) # Durable Postgres-backed queue depth (also updated on the worker process registry). EMBEDDING_QUEUE_JOBS = Gauge( - "cafa5_embedding_queue_jobs", + "embedding_queue_jobs", "Embedding jobs currently in each lifecycle state.", labelnames=("status",), registry=registry, ) RQ_QUEUE_LENGTH = Gauge( - "cafa5_rq_queue_length", + "rq_queue_length", "Redis/RQ queue length for embedding jobs.", labelnames=("queue",), registry=registry, From 0684756b07c41cfd0239d661b3372e380773fbdc Mon Sep 17 00:00:00 2001 From: Behrouz Mirabdi Date: Fri, 24 Jul 2026 16:52:47 +0200 Subject: [PATCH 4/9] refactor(monitoring): change monitoring metric name strings --- services/embedding-api/worker.py | 10 +++++----- services/go-prediction-api/main.py | 16 ++++++++-------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/services/embedding-api/worker.py b/services/embedding-api/worker.py index 4605b5f..101cb51 100644 --- a/services/embedding-api/worker.py +++ b/services/embedding-api/worker.py @@ -11,27 +11,27 @@ from prometheus_client import Counter, Gauge, Histogram EMBEDDING_JOBS_TOTAL = Counter( - "cafa5_embedding_jobs_total", + "embedding_jobs_total", "Total embedding jobs partitioned by terminal status and backend.", labelnames=("status", "backend"), ) EMBEDDING_QUEUE_JOBS = Gauge( - "cafa5_embedding_queue_jobs", + "embedding_queue_jobs", "Embedding jobs currently in each lifecycle state.", labelnames=("status",), ) EMBEDDING_JOB_DURATION_SECONDS = Histogram( - "cafa5_embedding_job_duration_seconds", + "embedding_job_duration_seconds", "Embedding job duration in seconds by terminal status and backend.", labelnames=("status", "backend"), ) EMBEDDING_SEQUENCES_PROCESSED_TOTAL = Counter( - "cafa5_embedding_sequences_processed_total", + "embedding_sequences_processed_total", "Number of embedded sequences processed by backend.", labelnames=("backend",), ) EMBEDDING_ARTIFACT_BYTES = Histogram( - "cafa5_embedding_artifact_bytes", + "embedding_artifact_bytes", "Embedding artifact output size in bytes partitioned by artifact name.", labelnames=("artifact_name",), buckets=(1024, 10 * 1024, 100 * 1024, 1024**2, 5 * 1024**2, 10 * 1024**2, float("inf")), diff --git a/services/go-prediction-api/main.py b/services/go-prediction-api/main.py index 8d6754b..764c4df 100644 --- a/services/go-prediction-api/main.py +++ b/services/go-prediction-api/main.py @@ -24,7 +24,7 @@ TERM_NAMES_PATH = APP_ROOT / "outputs" / "label_matrix_top500" / "term_names.npy" META_PATH = APP_ROOT / "outputs" / "splits" / "model_meta.json" -app = FastAPI(title="CAFA Inference API") +app = FastAPI(title="ProSeqGO Inference API") # Prometheus metrics for the GO prediction API registry = CollectorRegistry() # to store metrics @@ -34,43 +34,43 @@ # metrics for HTTP requests HTTP_REQUESTS_TOTAL = Counter( - "cafa5_http_requests_total", + "http_requests_total", "Total number of HTTP requests.", labelnames=("service", "route", "method", "status_code"), registry=registry, ) HTTP_REQUEST_DURATION_SECONDS = Histogram( - "cafa5_http_request_duration_seconds", + "http_request_duration_seconds", "HTTP request duration in seconds.", labelnames=("service", "route", "method", "status_code"), registry=registry, ) HTTP_IN_FLIGHT_REQUESTS = Gauge( - "cafa5_http_in_flight_requests", + "http_in_flight_requests", "Number of in-flight HTTP requests.", labelnames=("service",), registry=registry, ) INFERENCE_REQUESTS_TOTAL = Counter( - "cafa5_inference_requests_total", + "inference_requests_total", "Total inference requests partitioned by model version and status code.", labelnames=("model_version", "status_code"), registry=registry, ) INFERENCE_DURATION_SECONDS = Histogram( - "cafa5_inference_duration_seconds", + "inference_duration_seconds", "Inference runtime in seconds partitioned by model version.", labelnames=("model_version",), registry=registry, ) INFERENCE_INPUT_VALIDATION_FAILURES_TOTAL = Counter( - "cafa5_inference_input_validation_failures_total", + "inference_input_validation_failures_total", "Total inference input validation failures partitioned by reason.", labelnames=("reason",), registry=registry, ) INFERENCE_TOP_K_REQUESTS_TOTAL = Counter( - "cafa5_inference_top_k_requests_total", + "inference_top_k_requests_total", "Distribution of requested top_k values.", labelnames=("top_k",), registry=registry, From 172d7e9e5a7b62a3b1c5ff14b95c0e839d729b3f Mon Sep 17 00:00:00 2001 From: Behrouz Mirabdi Date: Fri, 24 Jul 2026 16:55:49 +0200 Subject: [PATCH 5/9] refactor(monitoring): change alert names and metrics names match to the metrics in the services --- monitoring/alerts.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/monitoring/alerts.yml b/monitoring/alerts.yml index d5b6969..a294ab9 100644 --- a/monitoring/alerts.yml +++ b/monitoring/alerts.yml @@ -1,8 +1,8 @@ groups: - - name: cafa5_simple_alerts + - name: proseqgo_simple_alerts interval: 30s rules: - - alert: Cafa5ServiceMetricsTargetDown + - alert: ProSeqGOServiceMetricsTargetDown expr: up{job=~"prometheus|embedding_api_metrics|embedding_worker_metrics|go_prediction_api_metrics|trainer_api_metrics|redis_exporter"} == 0 for: 2m labels: @@ -11,15 +11,15 @@ groups: summary: "Service scrape target down: {{ $labels.job }}" description: "{{ $labels.instance }} is down for >2m" - - alert: Cafa5HighHttp5xxRatio + - alert: ProSeqGOHighHttp5xxRatio expr: | ( - sum by (service) (rate(cafa5_http_requests_total{status_code=~"5.."}[5m])) + sum by (service) (rate(http_requests_total{status_code=~"5.."}[5m])) / - clamp_min(sum by (service) (rate(cafa5_http_requests_total[5m])), 0.001) + clamp_min(sum by (service) (rate(http_requests_total[5m])), 0.001) ) > 0.05 and - sum by (service) (rate(cafa5_http_requests_total[5m])) > 0.1 + sum by (service) (rate(http_requests_total[5m])) > 0.1 for: 10m labels: severity: warning @@ -27,8 +27,8 @@ groups: summary: "High 5xx ratio on {{ $labels.service }}" description: "5xx ratio >5% for 10m with enough traffic" - - alert: Cafa5EmbeddingQueueBacklogHigh - expr: cafa5_embedding_queue_jobs{status="queued"} > 20 + - alert: ProSeqGOEmbeddingQueueBacklogHigh + expr: embedding_queue_jobs{status="queued"} > 20 for: 10m labels: severity: warning @@ -36,11 +36,11 @@ groups: summary: "Embedding queue backlog high" description: "Queued embedding jobs >20 for 10m" - - alert: Cafa5EmbeddingWorkerDownWithBacklog + - alert: ProSeqGOEmbeddingWorkerDownWithBacklog expr: | up{job="embedding_worker_metrics"} == 0 and - cafa5_rq_queue_length{queue="embedding-jobs"} > 0 + rq_queue_length{queue="embedding-jobs"} > 0 for: 5m labels: severity: critical @@ -48,7 +48,7 @@ groups: summary: "Embedding RQ worker down while queue has jobs" description: "embedding-worker scrape is down and Redis queue embedding-jobs is non-empty" - - alert: Cafa5RedisDown + - alert: ProSeqGORedisDown expr: redis_up == 0 for: 2m labels: From b4b41c6926a2fa46458f3e5e575e7c9a9fb176eb Mon Sep 17 00:00:00 2001 From: Behrouz Mirabdi Date: Fri, 24 Jul 2026 17:04:52 +0200 Subject: [PATCH 6/9] refactor(monitoring): change the metrics names to be match with coresponding scripts in services --- .../grafana/dashboards/domain-pipelines.json | 494 ++++++++++++++++++ .../grafana/dashboards/service-health.json | 196 +++++++ 2 files changed, 690 insertions(+) create mode 100644 monitoring/grafana/dashboards/domain-pipelines.json create mode 100644 monitoring/grafana/dashboards/service-health.json diff --git a/monitoring/grafana/dashboards/domain-pipelines.json b/monitoring/grafana/dashboards/domain-pipelines.json new file mode 100644 index 0000000..18514ed --- /dev/null +++ b/monitoring/grafana/dashboards/domain-pipelines.json @@ -0,0 +1,494 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "id": 1, + "type": "text", + "title": "Embedding", + "gridPos": { "h": 2, "w": 24, "x": 0, "y": 0 }, + "options": { + "content": "## Embedding Pipeline", + "mode": "markdown" + } + }, + { + "id": 2, + "type": "timeseries", + "title": "Embedding Queue Depth by Status", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 2 }, + "targets": [ + { + "expr": "sum by (status) (embedding_queue_jobs)", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "none", "min": 0 }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" } + } + }, + { + "id": 3, + "type": "barchart", + "title": "Embedding Job Outcomes (15m)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 2 }, + "targets": [ + { + "expr": "sum by (status, backend) (increase(embedding_jobs_total{backend=~\"$backend\"}[15m]))", + "legendFormat": "{{backend}} {{status}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "none", "min": 0 }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" } + } + }, + { + "id": 4, + "type": "timeseries", + "title": "Embedding Job Duration P95 (s)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 2 }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le, status, backend) (rate(embedding_job_duration_seconds_bucket{backend=~\"$backend\"}[5m])))", + "legendFormat": "{{backend}} {{status}} p95", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "s", "min": 0 }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" } + } + }, + { + "id": 5, + "type": "timeseries", + "title": "Sequence Length Distribution (rate by bucket)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 10 }, + "targets": [ + { + "expr": "sum by (le, backend) (rate(embedding_sequence_length_bucket{backend=~\"$backend\"}[5m]))", + "legendFormat": "{{backend}} <= {{le}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "none", "min": 0 }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" } + } + }, + { + "id": 6, + "type": "timeseries", + "title": "Embedding Sequence Throughput (seq/s)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 10 }, + "targets": [ + { + "expr": "sum by (backend) (rate(embedding_sequences_processed_total{backend=~\"$backend\"}[5m]))", + "legendFormat": "{{backend}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "reqps", "min": 0 }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" } + } + }, + { + "id": 18, + "type": "timeseries", + "title": "RQ Queue Length (Redis)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 18 }, + "targets": [ + { + "expr": "rq_queue_length", + "legendFormat": "{{queue}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "none", "min": 0 }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" } + } + }, + { + "id": 19, + "type": "timeseries", + "title": "Redis Memory Used", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 18 }, + "targets": [ + { + "expr": "redis_memory_used_bytes", + "legendFormat": "used", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "bytes", "min": 0 }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" } + } + }, + + { + "id": 7, + "type": "text", + "title": "Training", + "gridPos": { "h": 2, "w": 24, "x": 0, "y": 26 }, + "options": { + "content": "## Training Pipeline", + "mode": "markdown" + } + }, + { + "id": 8, + "type": "timeseries", + "title": "Training Queue Depth by Status", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 28 }, + "targets": [ + { + "expr": "sum by (status) (training_queue_jobs)", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "none", "min": 0 }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" } + } + }, + { + "id": 9, + "type": "barchart", + "title": "Training Failure Reasons (15m)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 28 }, + "targets": [ + { + "expr": "sum by (reason) (increase(training_subprocess_failures_total{reason=~\"$reason\"}[15m]))", + "legendFormat": "{{reason}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "none", "min": 0 }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" } + } + }, + { + "id": 10, + "type": "timeseries", + "title": "Training Duration P95 by Mode/Status (s)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 28 }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le, mode, status) (rate(training_job_duration_seconds_bucket{mode=~\"$mode\"}[5m])))", + "legendFormat": "{{mode}} {{status}} p95", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "s", "min": 0 }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" } + } + }, + { + "id": 11, + "type": "timeseries", + "title": "Training Job Outcomes by Mode (15m)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 36 }, + "targets": [ + { + "expr": "sum by (mode, status) (increase(training_jobs_total{mode=~\"$mode\"}[15m]))", + "legendFormat": "{{mode}} {{status}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "none", "min": 0 }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" } + } + }, + + { + "id": 12, + "type": "text", + "title": "GO Inference", + "gridPos": { "h": 2, "w": 24, "x": 0, "y": 44 }, + "options": { + "content": "## GO Inference", + "mode": "markdown" + } + }, + { + "id": 13, + "type": "timeseries", + "title": "Inference Latency P95 by Model Version (s)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 46 }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le, model_version) (rate(inference_duration_seconds_bucket{model_version=~\"$model_version\"}[5m])))", + "legendFormat": "{{model_version}} p95", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "s", "min": 0 }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" } + } + }, + { + "id": 14, + "type": "timeseries", + "title": "Inference Request Rate by Model/Status", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 46 }, + "targets": [ + { + "expr": "sum by (model_version, status_code) (rate(inference_requests_total{model_version=~\"$model_version\"}[5m]))", + "legendFormat": "{{model_version}} {{status_code}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "reqps", "min": 0 }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" } + } + }, + { + "id": 15, + "type": "barchart", + "title": "Input Validation Failures by Reason (15m)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 46 }, + "targets": [ + { + "expr": "sum by (reason) (increase(inference_input_validation_failures_total{reason=~\"$reason\"}[15m]))", + "legendFormat": "{{reason}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "none", "min": 0 }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom" } + } + }, + { + "id": 16, + "type": "barchart", + "title": "Top-K Usage (1h)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 54 }, + "targets": [ + { + "expr": "sum by (top_k) (increase(inference_top_k_requests_total{top_k=~\"$top_k\"}[1h]))", + "legendFormat": "top_k={{top_k}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "none", "min": 0 }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" } + } + }, + { + "id": 17, + "type": "timeseries", + "title": "Inference Error Ratio (5m)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 54 }, + "targets": [ + { + "expr": "sum(rate(inference_requests_total{model_version=~\"$model_version\",status_code=~\"5..\"}[5m])) / clamp_min(sum(rate(inference_requests_total{model_version=~\"$model_version\"}[5m])), 1e-9)", + "legendFormat": "5xx ratio", + "refId": "A" + }, + { + "expr": "sum(rate(inference_requests_total{model_version=~\"$model_version\",status_code=~\"4..\"}[5m])) / clamp_min(sum(rate(inference_requests_total{model_version=~\"$model_version\"}[5m])), 1e-9)", + "legendFormat": "4xx ratio", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { "unit": "percentunit", "min": 0, "max": 1 }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" } + } + } + ], + "refresh": "30s", + "schemaVersion": 39, + "style": "dark", + "tags": ["cafa5", "domain", "mlops", "embedding", "training", "inference", "proseqgo"], + "templating": { + "list": [ + { + "name": "backend", + "type": "query", + "label": "Embedding Backend", + "hide": 0, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "definition": "label_values(embedding_jobs_total, backend)", + "query": { + "query": "label_values(embedding_jobs_total, backend)", + "refId": "BackendVar" + }, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { "text": "All", "value": ".*" } + }, + { + "name": "mode", + "type": "query", + "label": "Training Mode", + "hide": 0, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "definition": "label_values(training_jobs_total, mode)", + "query": { + "query": "label_values(training_jobs_total, mode)", + "refId": "ModeVar" + }, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { "text": "All", "value": ".*" } + }, + { + "name": "model_version", + "type": "query", + "label": "Model Version", + "hide": 0, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "definition": "label_values(inference_requests_total, model_version)", + "query": { + "query": "label_values(inference_requests_total, model_version)", + "refId": "ModelVerVar" + }, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { "text": "All", "value": ".*" } + }, + { + "name": "reason", + "type": "query", + "label": "Failure Reason", + "hide": 0, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "definition": "label_values(training_subprocess_failures_total, reason)", + "query": { + "query": "label_values(training_subprocess_failures_total, reason)", + "refId": "ReasonVar" + }, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { "text": "All", "value": ".*" } + }, + { + "name": "top_k", + "type": "query", + "label": "Top-K", + "hide": 0, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "definition": "label_values(inference_top_k_requests_total, top_k)", + "query": { + "query": "label_values(inference_top_k_requests_total, top_k)", + "refId": "TopKVar" + }, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { "text": "All", "value": ".*" } + } + ] + }, + "time": { "from": "now-6h", "to": "now" }, + "timepicker": {}, + "timezone": "", + "title": "ProSeqGO Domain Pipelines", + "uid": "proseqgo-domain-pipelines", + "version": 1, + "weekStart": "" + } \ No newline at end of file diff --git a/monitoring/grafana/dashboards/service-health.json b/monitoring/grafana/dashboards/service-health.json new file mode 100644 index 0000000..bede542 --- /dev/null +++ b/monitoring/grafana/dashboards/service-health.json @@ -0,0 +1,196 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Targets Up (metrics jobs)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 6, "w": 6, "x": 0, "y": 0 }, + "targets": [ + { + "expr": "sum(up{job=~\".*_metrics\"})", + "refId": "A" + } + ], + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value" + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 1 } + ] + } + }, + "overrides": [] + } + }, + { + "id": 2, + "type": "timeseries", + "title": "Request Rate (req/s)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 18, "x": 6, "y": 0 }, + "targets": [ + { + "expr": "sum by (service, route) (rate(http_requests_total{service=~\"$service\",route=~\"$route\",status_code=~\"$status_code\"}[5m]))", + "legendFormat": "{{service}} {{route}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "reqps" }, + "overrides": [] + }, + "options": { "legend": { "displayMode": "table", "placement": "bottom" } } + }, + { + "id": 3, + "type": "timeseries", + "title": "Error Ratio (5xx / total)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 6 }, + "targets": [ + { + "expr": "sum(rate(http_requests_total{service=~\"$service\",route=~\"$route\",status_code=~\"5..\"}[5m])) / clamp_min(sum(rate(cafa5_http_requests_total{service=~\"$service\",route=~\"$route\"}[5m])), 1e-9)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "percentunit", "min": 0, "max": 1 }, + "overrides": [] + }, + "options": { "legend": { "displayMode": "list", "placement": "bottom" } } + }, + { + "id": 4, + "type": "timeseries", + "title": "P95 Latency (s)", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 6 }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le, service, route) (rate(http_request_duration_seconds_bucket{service=~\"$service\",route=~\"$route\",status_code=~\"$status_code\"}[5m])))", + "legendFormat": "{{service}} {{route}} p95", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "s" }, + "overrides": [] + }, + "options": { "legend": { "displayMode": "table", "placement": "bottom" } } + }, + { + "id": 5, + "type": "timeseries", + "title": "In-flight Requests", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 6 }, + "targets": [ + { + "expr": "sum by (service) (http_in_flight_requests{service=~\"$service\"})", + "legendFormat": "{{service}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "none", "min": 0 }, + "overrides": [] + }, + "options": { "legend": { "displayMode": "list", "placement": "bottom" } } + } + ], + "refresh": "30s", + "schemaVersion": 39, + "style": "dark", + "tags": ["cafa5", "operations", "http", "proseqgo"], + "templating": { + "list": [ + { + "name": "service", + "type": "query", + "label": "Service", + "hide": 0, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "definition": "label_values(http_requests_total, service)", + "query": { + "query": "label_values(http_requests_total, service)", + "refId": "ServiceVar" + }, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { "text": "All", "value": ".*" } + }, + { + "name": "route", + "type": "query", + "label": "Route", + "hide": 0, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "definition": "label_values(http_requests_total{service=~\"$service\"}, route)", + "query": { + "query": "label_values(http_requests_total{service=~\"$service\"}, route)", + "refId": "RouteVar" + }, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { "text": "All", "value": ".*" } + }, + { + "name": "status_code", + "type": "query", + "label": "Status Code", + "hide": 0, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "definition": "label_values(http_requests_total{service=~\"$service\",route=~\"$route\"}, status_code)", + "query": { + "query": "label_values(http_requests_total{service=~\"$service\",route=~\"$route\"}, status_code)", + "refId": "StatusVar" + }, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { "text": "All", "value": ".*" } + } + ] + }, + "time": { "from": "now-6h", "to": "now" }, + "timepicker": {}, + "timezone": "", + "title": "ProSeqGO Service Health", + "uid": "proseqgo-service-health", + "version": 1, + "weekStart": "" + } \ No newline at end of file From 801520cdbddbd64dfe249dcef0c1b41518a62aef Mon Sep 17 00:00:00 2001 From: Behrouz Mirabdi Date: Fri, 24 Jul 2026 20:53:45 +0200 Subject: [PATCH 7/9] ci: add PR lint, unit tests, and image build pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lay down Phase 0–1C CI foundations: Makefile quality targets, Ruff on PRs, fast Docker-free unit tests, and parallel Buildx image builds with GHA cache (build only, no registry push yet). Co-authored-by: Cursor --- .dockerignore | 27 +++++ .github/CI.md | 45 ++++++++ .github/workflows/ci.yml | 109 ++++++++++++++++++ Makefile | 66 ++++++++++- README.md | 18 +++ .../docker_embedding/Dockerfile.embedding-api | 5 +- docker/docker_go_term/Dockerfile.api | 5 +- docker/docker_training/Dockerfile.training | 10 +- pyproject.toml | 11 ++ scripts/embed_sequences.py | 51 +------- services/embedding-api/embedder.py | 6 +- .../go-prediction-api/embedding_validation.py | 17 +++ .../go-prediction-api/predictor_service.py | 12 +- services/streamlit-ui/app.py | 59 ++-------- services/streamlit-ui/validation.py | 50 ++++++++ src/inference/predictor.py | 1 - src/preprocess/sequences.py | 54 +++++++++ src/training/trainer.py | 1 - tests/unit/README.md | 19 +++ tests/unit/conftest.py | 62 ++++++++++ tests/unit/test_api_schemas.py | 74 ++++++++++++ tests/unit/test_config.py | 45 ++++++++ tests/unit/test_embedding_validation.py | 23 ++++ tests/unit/test_sequences.py | 46 ++++++++ tests/unit/test_ui_validation.py | 59 ++++++++++ 25 files changed, 755 insertions(+), 120 deletions(-) create mode 100644 .dockerignore create mode 100644 .github/CI.md create mode 100644 .github/workflows/ci.yml create mode 100644 services/go-prediction-api/embedding_validation.py create mode 100644 services/streamlit-ui/validation.py create mode 100644 src/preprocess/sequences.py create mode 100644 tests/unit/README.md create mode 100644 tests/unit/conftest.py create mode 100644 tests/unit/test_api_schemas.py create mode 100644 tests/unit/test_config.py create mode 100644 tests/unit/test_embedding_validation.py create mode 100644 tests/unit/test_sequences.py create mode 100644 tests/unit/test_ui_validation.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..aea6d25 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,27 @@ +# Keep Compose/CI build context small. Data and secrets are mounted at runtime. +.git +.gitignore +.github +.venv +venv +__pycache__/ +*.pyc +*.pyo +*.npy +*.pt +*.pth +*.safetensors +data/ +outputs/ +backups/ +archives/ +notes/ +notebooks/ +docs/ +tests/ +.cursor/ +.idea/ +.vscode/ +.env +.env.* +!.env.example diff --git a/.github/CI.md b/.github/CI.md new file mode 100644 index 0000000..930c90d --- /dev/null +++ b/.github/CI.md @@ -0,0 +1,45 @@ +# Continuous Integration notes (ProSeqGo) + +## What runs today + +| Trigger | Job | Command | +|---------|-----|---------| +| `pull_request`, `push` to `main` | Lint | `make lint` → `ruff check src services scripts` | +| `pull_request`, `push` to `main` | Unit tests (after lint) | `make test` → `pytest tests/unit -q` | +| `pull_request`, `push` to `main` | Image builds (after lint, parallel) | Buildx build of all 5 product images (**no push**) | + +Unit tests are **Docker/GPU/network free**. They cover sequence normalization, config loading, API schema bounds, embedding vector validation, and UI input gates. + +### Image builds (Phase 1C) + +| Image | Dockerfile | +|-------|------------| +| `proseqgo-embedding-api` | `docker/docker_embedding/Dockerfile.embedding-api` | +| `proseqgo-go-prediction-api` | `docker/docker_go_term/Dockerfile.api` | +| `proseqgo-streamlit-ui` | `docker/docker_streamlit/Dockerfile.streamlit` | +| `proseqgo-trainer-api` | `docker/docker_training/Dockerfile.training` | +| `proseqgo-mlflow` | `docker/docker_mlflow/Dockerfile` | + +- PR/`main`: **build only** (GHA layer cache, `outputs: type=cacheonly`) +- Torch images use **CPU wheels** in CI (`TORCH_INDEX_URL=.../cpu`) for smaller/faster builds; local Compose still defaults to CUDA index +- Local: `make build-images` (or `TORCH_INDEX_URL=https://download.pytorch.org/whl/cpu make build-images`) + +## Registry (Phase 2) + +Images will publish to **GHCR** (`ghcr.io/BehRoooz/...`) with immutable `sha-` tags. Not wired yet. + +## Compose / secrets in CI (Phase 3) + +Workflows must never commit real secrets. Pattern: + +```bash +make ci-env # copies .env.example → .env if missing +``` + +Use throwaway passwords from `.env.example` only inside ephemeral CI runners. + +## Explicit non-goals for CI + +- No training / GPU / full retrain jobs in PR or `main` CI +- No Training API profile in automated smoke (serving stack only) +- No registry push yet (Phase 2); Compose smoke lands in Phase 3 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..34d16af --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,109 @@ +name: CI + +on: + pull_request: + push: + branches: [main] + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + actions: write + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + # Lint-only install: do not pip install the project (would pull torch/HF). + - name: Install Ruff + run: pip install "ruff>=0.1" + + - name: Ruff check + run: make lint + + test: + name: Unit tests + runs-on: ubuntu-latest + needs: lint + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + # Slim install: unit tests must not need torch/transformers/streamlit. + - name: Install test dependencies + run: pip install "pytest>=7.0" "pydantic>=2.0" "pyyaml>=6.0" "numpy>=1.24" + + - name: Run unit tests + run: make test + + # Phase 1C: prove all product images build. No registry push (Phase 2). + build-images: + name: Build ${{ matrix.name }} + runs-on: ubuntu-latest + needs: lint + strategy: + fail-fast: false + matrix: + include: + - name: embedding-api + dockerfile: docker/docker_embedding/Dockerfile.embedding-api + image: proseqgo-embedding-api + build_args: TORCH_INDEX_URL=https://download.pytorch.org/whl/cpu + - name: go-prediction-api + dockerfile: docker/docker_go_term/Dockerfile.api + image: proseqgo-go-prediction-api + build_args: TORCH_INDEX_URL=https://download.pytorch.org/whl/cpu + - name: streamlit-ui + dockerfile: docker/docker_streamlit/Dockerfile.streamlit + image: proseqgo-streamlit-ui + build_args: "" + - name: trainer-api + dockerfile: docker/docker_training/Dockerfile.training + image: proseqgo-trainer-api + build_args: TORCH_INDEX_URL=https://download.pytorch.org/whl/cpu + - name: mlflow + dockerfile: docker/docker_mlflow/Dockerfile + image: proseqgo-mlflow + build_args: "" + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Free disk space + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \ + /usr/local/share/powershell /usr/share/swift /usr/local/graalvm \ + /usr/local/.ghcup || true + df -h + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build image (no push) + uses: docker/build-push-action@v6 + with: + context: . + file: ${{ matrix.dockerfile }} + push: false + outputs: type=cacheonly + tags: ${{ matrix.image }}:sha-${{ github.sha }} + build-args: ${{ matrix.build_args }} + cache-from: type=gha,scope=build-${{ matrix.name }} + cache-to: type=gha,mode=max,scope=build-${{ matrix.name }} diff --git a/Makefile b/Makefile index 9515f5e..29b5d3c 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,18 @@ -.PHONY: help up down training-up training-down monitoring-up monitoring-down +.PHONY: help up down restart training-up training-down monitoring-up monitoring-down \ + up-all down-all lint test build-images smoke ci-env + +# Product Python paths linted in CI (Phase 1A). Expand later if needed. +LINT_PATHS := src services scripts +PYTHON ?= python3 + +# Image names match docker-compose.yml local tags. +EMBEDDING_IMAGE ?= proseqgo-embedding-api:local +GO_PRED_IMAGE ?= proseqgo-go-prediction-api:local +STREAMLIT_IMAGE ?= proseqgo-streamlit-ui:local +TRAINER_IMAGE ?= proseqgo-trainer-api:local +MLFLOW_IMAGE ?= proseqgo-mlflow:local +# Local default matches GPU Compose; CI passes cpu index via TORCH_INDEX_URL. +TORCH_INDEX_URL ?= https://download.pytorch.org/whl/cu132 help: @echo "Available targets:" @@ -8,6 +22,11 @@ help: @echo " make training-down - Stop services started with the training profile" @echo " make monitoring-up - Start services with the monitoring profile" @echo " make monitoring-down - Stop services started with the monitoring profile" + @echo " make lint - Ruff check on src/ services/ scripts/" + @echo " make test - Unit tests (tests/unit; Phase 1B)" + @echo " make build-images - Build all product Docker images" + @echo " make smoke - Run Compose smoke scripts (stack must be up)" + @echo " make ci-env - Copy .env.example -> .env for local/CI Compose" up: docker compose up -d --build @@ -26,13 +45,52 @@ training-down: docker compose --profile training down monitoring-up: - docker compose --profile monitoring up -d + docker compose --profile monitoring up -d --build monitoring-down: docker compose --profile monitoring down up-all: - docker compose --profile monitoring up -d --build + docker compose --profile monitoring --profile training up -d --build down-all: - docker compose down \ No newline at end of file + docker compose --profile monitoring --profile training down + +# --- CI / quality (same commands locally and in GitHub Actions) --- + +ci-env: + @test -f .env.example || (echo "Missing .env.example"; exit 1) + @if [ -f .env ]; then \ + echo ".env already present (left unchanged)"; \ + else \ + cp .env.example .env; \ + echo "Created .env from .env.example"; \ + fi + +lint: + ruff check $(LINT_PATHS) + +test: + $(PYTHON) -m pytest tests/unit -q + +build-images: + docker build \ + --build-arg TORCH_INDEX_URL=$(TORCH_INDEX_URL) \ + -f docker/docker_embedding/Dockerfile.embedding-api \ + -t $(EMBEDDING_IMAGE) . + docker build \ + --build-arg TORCH_INDEX_URL=$(TORCH_INDEX_URL) \ + -f docker/docker_go_term/Dockerfile.api \ + -t $(GO_PRED_IMAGE) . + docker build -f docker/docker_streamlit/Dockerfile.streamlit -t $(STREAMLIT_IMAGE) . + docker build \ + --build-arg TORCH_INDEX_URL=$(TORCH_INDEX_URL) \ + -f docker/docker_training/Dockerfile.training \ + -t $(TRAINER_IMAGE) . + docker build -f docker/docker_mlflow/Dockerfile -t $(MLFLOW_IMAGE) . + +# Expects default Compose stack already healthy. Does not start training/GPU jobs. +smoke: + ./tests/smoke/smoke_embedding_api.sh + @echo "Optional: MLFLOW_TRACKING_URI=http://127.0.0.1/mlflow python tests/smoke/mlflow_smoke_test.py" + @echo "Optional (heavier): ./tests/smoke/test_embedding_worker_crash_recovery.sh" diff --git a/README.md b/README.md index bdb02dc..eb501a7 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,11 @@ CAFA-5-MLOps-solution/ ├── outputs/ # Splits, labels, checkpoints, artifacts, submissions ├── scripts/ # CLI pipeline entrypoints (preprocess/train/evaluate/predict) ├── tests/ +│ ├── unit/ # Fast pytest suite (no Docker/GPU; CI Phase 1B) │ └── smoke/ # Compose smoke/acceptance checks (not unit tests) +├── .github/ +│ ├── workflows/ci.yml # PR/main CI (lint today; tests/images later) +│ └── CI.md # CI scope, GHCR plan, non-goals ├── services/ │ ├── embedding-api/ # Async embedding jobs + sequence->GO orchestration endpoint │ ├── go-prediction-api/ # Embedding->GO inference API @@ -459,8 +463,22 @@ make training-up make training-down make monitoring-up make monitoring-down +make lint # Ruff on src/ services/ scripts/ +make test # Unit tests (tests/unit) +make build-images # Build all five product images +make smoke # Smoke scripts (Compose stack must already be up) +make ci-env # Copy .env.example → .env if missing ``` +## CI (GitHub Actions) + +PR and `main` pushes run **lint**, **unit tests**, and **parallel image builds** (no registry push yet). See [`.github/CI.md`](.github/CI.md). + +- Registry target for later phases: **GHCR** +- CI does **not** run training/GPU/retrain jobs +- Compose in CI will use `make ci-env` (`.env.example` only)—never commit real secrets +- Local image rebuild: `make build-images` + ## Service-Specific Documentation - `services/embedding-api/README.md` diff --git a/docker/docker_embedding/Dockerfile.embedding-api b/docker/docker_embedding/Dockerfile.embedding-api index a6ea4dd..6c71c72 100644 --- a/docker/docker_embedding/Dockerfile.embedding-api +++ b/docker/docker_embedding/Dockerfile.embedding-api @@ -4,6 +4,9 @@ ENV PYTHONUNBUFFERED=1 \ PIP_NO_CACHE_DIR=1 \ HF_HOME=/app/data/hf_cache +# Override in CI with CPU wheels: --build-arg TORCH_INDEX_URL=https://download.pytorch.org/whl/cpu +ARG TORCH_INDEX_URL=https://download.pytorch.org/whl/cu132 + WORKDIR /app RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -12,7 +15,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ COPY requirements.txt /app/requirements.txt RUN pip install --upgrade pip && \ - pip install torch torchvision --index-url https://download.pytorch.org/whl/cu132 && \ + pip install torch torchvision --index-url ${TORCH_INDEX_URL} && \ pip install -r /app/requirements.txt # Copy core project code used by the worker / embedder. diff --git a/docker/docker_go_term/Dockerfile.api b/docker/docker_go_term/Dockerfile.api index 45bf5e4..ef217bc 100644 --- a/docker/docker_go_term/Dockerfile.api +++ b/docker/docker_go_term/Dockerfile.api @@ -5,6 +5,9 @@ ENV PYTHONDONTWRITEBYTECODE=1 \ PIP_NO_CACHE_DIR=1 \ PYTHONPATH=/app:/app/services/go-prediction-api +# Override in CI with CPU wheels: --build-arg TORCH_INDEX_URL=https://download.pytorch.org/whl/cpu +ARG TORCH_INDEX_URL=https://download.pytorch.org/whl/cu132 + WORKDIR /app RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -14,7 +17,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ COPY requirements.txt /app/ RUN pip install --upgrade pip "setuptools<82" wheel && \ - pip install torch torchvision --index-url https://download.pytorch.org/whl/cu132 && \ + pip install torch torchvision --index-url ${TORCH_INDEX_URL} && \ pip install -r /app/requirements.txt COPY src /app/src diff --git a/docker/docker_training/Dockerfile.training b/docker/docker_training/Dockerfile.training index 52ab193..6492000 100644 --- a/docker/docker_training/Dockerfile.training +++ b/docker/docker_training/Dockerfile.training @@ -3,6 +3,9 @@ FROM python:3.12-slim ENV PYTHONUNBUFFERED=1 \ PIP_NO_CACHE_DIR=1 +# Override in CI with CPU wheels: --build-arg TORCH_INDEX_URL=https://download.pytorch.org/whl/cpu +ARG TORCH_INDEX_URL=https://download.pytorch.org/whl/cu132 + WORKDIR /app RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -11,21 +14,20 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ COPY requirements.txt /app/requirements.txt RUN pip install --upgrade pip && \ - pip install torch torchvision --index-url https://download.pytorch.org/whl/cu132 && \ + pip install torch torchvision --index-url ${TORCH_INDEX_URL} && \ pip install -r /app/requirements.txt COPY src /app/src COPY scripts /app/scripts COPY configs /app/configs -COPY data/cafa-5-protein-function-prediction/Train/train_terms.tsv /app/data/cafa-5-protein-function-prediction/Train/train_terms.tsv # Copy the Training API service. COPY services/training-api /app/services/training-api -# Runtime directories (mounted in docker run / compose). +# Runtime directories (mounted in docker run / compose). CAFA labels/FASTA live under ./data. RUN mkdir -p /app/data /app/outputs EXPOSE 8000 # Same pattern as docker_embedding/Dockerfile.embedding-api: invoke uvicorn directly (no python ENTRYPOINT). -CMD ["uvicorn", "main:app", "--app-dir", "services/training-api", "--host", "0.0.0.0", "--port", "8000"] \ No newline at end of file +CMD ["uvicorn", "main:app", "--app-dir", "services/training-api", "--host", "0.0.0.0", "--port", "8000"] diff --git a/pyproject.toml b/pyproject.toml index 8db3720..a2e8f36 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,9 @@ dependencies = [ dev = [ "pytest>=7.0", "ruff>=0.1", + "pydantic>=2.0", + "pyyaml>=6.0", + "numpy>=1.24", ] [tool.setuptools.packages.find] @@ -38,3 +41,11 @@ include = ["src*"] line-length = 100 target-version = "py310" +# Phase 1A: classic pyflakes/pycodestyle starter set. Expand in later CI phases. +[tool.ruff.lint] +select = ["E4", "E7", "E9", "F"] + +[tool.pytest.ini_options] +testpaths = ["tests/unit"] +pythonpath = ["."] + diff --git a/scripts/embed_sequences.py b/scripts/embed_sequences.py index c5355e8..8bbcb13 100644 --- a/scripts/embed_sequences.py +++ b/scripts/embed_sequences.py @@ -58,52 +58,11 @@ }, } -AA_ALPHABET: set[str] = set("ACDEFGHIKLMNPQRSTVWY") -AA_REMAP: dict[str, str] = { - # common non-canonical amino acids -> unknown - "U": "X", # selenocysteine - "O": "X", # pyrrolysine - "B": "X", # aspartic acid or asparagine - "Z": "X", # glutamic acid or glutamine - "J": "X", # leucine or isoleucine - "X": "X", # unknown -} - - -def extract_protein_id(header_line: str) -> str: - """Extract CAFA/UniProt-like EntryID from FASTA headers.""" - h = header_line.strip().lstrip(">") - if "|" in h: - parts = h.split("|") - # Common UniProt format: sp|ENTRY|... - if len(parts) >= 2 and parts[1]: - return parts[1] - return h.split()[0] - - -def normalize_sequence(seq: str) -> str: - """Uppercase + remap rare tokens to `X` and validate characters.""" - seq = seq.strip().upper().replace(" ", "").replace("\n", "").replace("\t", "") - remapped: list[str] = [] - invalid_count = 0 - for aa in seq: - if aa in AA_ALPHABET: - remapped.append(aa) - elif aa in AA_REMAP: - remapped.append(AA_REMAP[aa]) - else: - invalid_count += 1 - remapped.append("X") - if invalid_count: - logger.warning("Remapped %d invalid amino acids to X", invalid_count) - return "".join(remapped) - - -def format_for_tokenizer(seq: str, tokenizer_mode: str) -> str: - """Transform sequence for the tokenizer (some models expect spaces).""" - if tokenizer_mode == "space_separated": - return " ".join(seq) - return seq +from src.preprocess.sequences import ( # noqa: E402 + extract_protein_id, + format_for_tokenizer, + normalize_sequence, +) def residue_mean_pool( diff --git a/services/embedding-api/embedder.py b/services/embedding-api/embedder.py index d6d1540..e508473 100644 --- a/services/embedding-api/embedder.py +++ b/services/embedding-api/embedder.py @@ -4,15 +4,15 @@ import sys import numpy as np -import torch # Make repo root importable when running uvicorn with --app-dir services/embedding-api REPO_ROOT = Path(__file__).resolve().parents[2] if str(REPO_ROOT) not in sys.path: sys.path.insert(0, str(REPO_ROOT)) -from scripts.embed_sequences import HF_MODEL_REGISTRY, embed_sequences, normalize_sequence -from src.utils import get_device +from scripts.embed_sequences import HF_MODEL_REGISTRY, embed_sequences # noqa: E402 +from src.preprocess.sequences import normalize_sequence # noqa: E402 +from src.utils import get_device # noqa: E402 _MODEL_CACHE: dict[str, tuple[object, object]] = {} diff --git a/services/go-prediction-api/embedding_validation.py b/services/go-prediction-api/embedding_validation.py new file mode 100644 index 0000000..f5bc55d --- /dev/null +++ b/services/go-prediction-api/embedding_validation.py @@ -0,0 +1,17 @@ +"""Pure embedding input checks for the GO prediction API (no torch).""" + +from __future__ import annotations + +import numpy as np + + +def validate_embedding(embedding: list[float] | np.ndarray, expected_dim: int) -> np.ndarray: + arr = np.asarray(embedding, dtype=np.float32) + + if arr.ndim != 1: + raise ValueError("embedding must be a 1-dimensional list") + + if arr.shape[0] != expected_dim: + raise ValueError(f"embedding must have length {expected_dim}") + + return arr diff --git a/services/go-prediction-api/predictor_service.py b/services/go-prediction-api/predictor_service.py index c55c987..fc4b303 100644 --- a/services/go-prediction-api/predictor_service.py +++ b/services/go-prediction-api/predictor_service.py @@ -5,17 +5,7 @@ import numpy as np import torch - -def validate_embedding(embedding: list[float] | np.ndarray, expected_dim: int) -> np.ndarray: - arr = np.asarray(embedding, dtype=np.float32) - - if arr.ndim != 1: - raise ValueError("embedding must be a 1-dimensional list") - - if arr.shape[0] != expected_dim: - raise ValueError(f"embedding must have length {expected_dim}") - - return arr +from embedding_validation import validate_embedding def predict_top_k( diff --git a/services/streamlit-ui/app.py b/services/streamlit-ui/app.py index 91a6671..080dede 100644 --- a/services/streamlit-ui/app.py +++ b/services/streamlit-ui/app.py @@ -1,6 +1,5 @@ from __future__ import annotations -import re import os from typing import Any @@ -10,8 +9,16 @@ from requests.auth import HTTPBasicAuth from requests.exceptions import RequestException +from validation import ( + MAX_FASTA_UPLOAD_BYTES, + normalize_sequence, + validate_fasta_upload, + validate_gateway_auth, + validate_sequence, +) + -PROJECT_TITLE = "CAFA-5 MLOps Solution" +PROJECT_TITLE = "ProSeqGO" PROJECT_DESCRIPTION = ( "Interactive sequence-to-GO inference UI backed by the embedding and GO " "prediction APIs through the NGINX gateway." @@ -20,12 +27,10 @@ PREDICT_SEQUENCES_ENDPOINT = "/api/v1/predict-go-from-sequences" PREDICT_FASTA_ENDPOINT = "/api/v1/predict-go-from-fasta" MAX_TOP_K = 500 -MAX_FASTA_UPLOAD_BYTES = 5 * 1024 * 1024 # must match embedding-api config + nginx route SEQUENCE_TIMEOUT_SECONDS = 600 FASTA_TIMEOUT_SECONDS = 1800 PREDICTION_MODE_SEQUENCE = "Prediction with sequence" PREDICTION_MODE_FASTA = "Prediction with FASTA" -AA_PATTERN = re.compile(r"^[ACDEFGHIKLMNPQRSTVWY]+$") WORKFLOW_DOT = """ digraph cafa5 { rankdir=LR; @@ -47,48 +52,6 @@ """.strip() -def normalize_sequence(raw_sequence: str) -> str: - compact = re.sub(r"\s+", "", raw_sequence or "") - return compact.upper() - - -def validate_sequence(sequence: str) -> tuple[bool, str]: - if not sequence: - return False, "Sequence is empty after whitespace cleanup." - if not AA_PATTERN.fullmatch(sequence): - return ( - False, - "Sequence includes invalid symbols. Allowed amino acids: ACDEFGHIKLMNPQRSTVWY.", - ) - return True, "" - - -def validate_fasta_upload(file_bytes: bytes, filename: str) -> tuple[bool, str]: - if not file_bytes: - return False, "Uploaded FASTA file is empty." - if len(file_bytes) > MAX_FASTA_UPLOAD_BYTES: - max_mb = MAX_FASTA_UPLOAD_BYTES // (1024 * 1024) - return False, f"FASTA file exceeds the {max_mb} MB upload limit." - try: - fasta_text = file_bytes.decode("utf-8") - except UnicodeDecodeError: - return False, "FASTA file must be valid UTF-8 text." - if not fasta_text.strip(): - return False, "Uploaded FASTA file contains no sequence data." - record_count = sum(1 for line in fasta_text.splitlines() if line.startswith(">")) - if record_count == 0: - return False, "FASTA file has no records (lines starting with '>')." - return True, "" - - -def validate_gateway_auth(gateway_base_url: str, username: str, password: str) -> str | None: - if not gateway_base_url.strip(): - return "Gateway base URL is required." - if not username.strip() or not password: - return "Both API username and password are required." - return None - - def parse_error_message(response: requests.Response) -> str: try: payload = response.json() @@ -228,8 +191,8 @@ def _render_shared_connection_fields( def main() -> None: - st.set_page_config(page_title="CAFA-5 UI", page_icon="🧬", layout="wide") - st.title("🧬 CAFA-5 Sequence-to-GO Prediction UI") + st.set_page_config(page_title="ProSeqGO", page_icon="🧬", layout="wide") + st.title("🧬 ProSeqGO: Protein Sequence to GO Prediction") st.write(PROJECT_DESCRIPTION) st.subheader("Platform Links") diff --git a/services/streamlit-ui/validation.py b/services/streamlit-ui/validation.py new file mode 100644 index 0000000..a2378ac --- /dev/null +++ b/services/streamlit-ui/validation.py @@ -0,0 +1,50 @@ +"""UI input validation helpers (no Streamlit dependency).""" + +from __future__ import annotations + +import re + +AA_PATTERN = re.compile(r"^[ACDEFGHIKLMNPQRSTVWY]+$") +MAX_FASTA_UPLOAD_BYTES = 5 * 1024 * 1024 # must match embedding-api config + nginx route + + +def normalize_sequence(raw_sequence: str) -> str: + compact = re.sub(r"\s+", "", raw_sequence or "") + return compact.upper() + + +def validate_sequence(sequence: str) -> tuple[bool, str]: + if not sequence: + return False, "Sequence is empty after whitespace cleanup." + if not AA_PATTERN.fullmatch(sequence): + return ( + False, + "Sequence includes invalid symbols. Allowed amino acids: ACDEFGHIKLMNPQRSTVWY.", + ) + return True, "" + + +def validate_fasta_upload(file_bytes: bytes, filename: str) -> tuple[bool, str]: + if not file_bytes: + return False, "Uploaded FASTA file is empty." + if len(file_bytes) > MAX_FASTA_UPLOAD_BYTES: + max_mb = MAX_FASTA_UPLOAD_BYTES // (1024 * 1024) + return False, f"FASTA file exceeds the {max_mb} MB upload limit." + try: + fasta_text = file_bytes.decode("utf-8") + except UnicodeDecodeError: + return False, "FASTA file must be valid UTF-8 text." + if not fasta_text.strip(): + return False, "Uploaded FASTA file contains no sequence data." + record_count = sum(1 for line in fasta_text.splitlines() if line.startswith(">")) + if record_count == 0: + return False, "FASTA file has no records (lines starting with '>')." + return True, "" + + +def validate_gateway_auth(gateway_base_url: str, username: str, password: str) -> str | None: + if not gateway_base_url.strip(): + return "Gateway base URL is required." + if not username.strip() or not password: + return "Both API username and password are required." + return None diff --git a/src/inference/predictor.py b/src/inference/predictor.py index 63ddbc1..aae75fc 100644 --- a/src/inference/predictor.py +++ b/src/inference/predictor.py @@ -15,7 +15,6 @@ from src.preprocess.dataset import ProteinSequenceDataset from src.models import build_model from src.utils import get_device -from src.config import Config logger = logging.getLogger("cafa5") diff --git a/src/preprocess/sequences.py b/src/preprocess/sequences.py new file mode 100644 index 0000000..56ea3fb --- /dev/null +++ b/src/preprocess/sequences.py @@ -0,0 +1,54 @@ +"""Protein sequence helpers shared by CLI embedding and APIs.""" + +from __future__ import annotations + +import logging + +logger = logging.getLogger("cafa5") + +AA_ALPHABET: set[str] = set("ACDEFGHIKLMNPQRSTVWY") +AA_REMAP: dict[str, str] = { + # common non-canonical amino acids -> unknown + "U": "X", # selenocysteine + "O": "X", # pyrrolysine + "B": "X", # aspartic acid or asparagine + "Z": "X", # glutamic acid or glutamine + "J": "X", # leucine or isoleucine + "X": "X", # unknown +} + + +def extract_protein_id(header_line: str) -> str: + """Extract CAFA/UniProt-like EntryID from FASTA headers.""" + h = header_line.strip().lstrip(">") + if "|" in h: + parts = h.split("|") + # Common UniProt format: sp|ENTRY|... + if len(parts) >= 2 and parts[1]: + return parts[1] + return h.split()[0] + + +def normalize_sequence(seq: str) -> str: + """Uppercase + remap rare tokens to `X` and validate characters.""" + seq = seq.strip().upper().replace(" ", "").replace("\n", "").replace("\t", "") + remapped: list[str] = [] + invalid_count = 0 + for aa in seq: + if aa in AA_ALPHABET: + remapped.append(aa) + elif aa in AA_REMAP: + remapped.append(AA_REMAP[aa]) + else: + invalid_count += 1 + remapped.append("X") + if invalid_count: + logger.warning("Remapped %d invalid amino acids to X", invalid_count) + return "".join(remapped) + + +def format_for_tokenizer(seq: str, tokenizer_mode: str) -> str: + """Transform sequence for the tokenizer (some models expect spaces).""" + if tokenizer_mode == "space_separated": + return " ".join(seq) + return seq diff --git a/src/training/trainer.py b/src/training/trainer.py index de3627a..38e658d 100644 --- a/src/training/trainer.py +++ b/src/training/trainer.py @@ -3,7 +3,6 @@ from __future__ import annotations import logging -from pathlib import Path from typing import Any import numpy as np diff --git a/tests/unit/README.md b/tests/unit/README.md new file mode 100644 index 0000000..e91724b --- /dev/null +++ b/tests/unit/README.md @@ -0,0 +1,19 @@ +# Unit tests + +Fast tests that run without Docker, GPU, or network. + +```bash +# from a venv with pytest, pydantic, pyyaml, numpy: +make test +# or: pytest tests/unit -q +``` + +| Module | What it protects | +|--------|------------------| +| `test_sequences.py` | Shared AA normalize / FASTA id / tokenizer formatting | +| `test_config.py` | YAML config load + embedding-dim validation | +| `test_api_schemas.py` | Embedding/GO request schema bounds | +| `test_embedding_validation.py` | GO API embedding vector shape/dim checks | +| `test_ui_validation.py` | Public UI sequence/FASTA/auth gates | + +Smoke / acceptance checks against a live Compose stack live in `tests/smoke/` and are **not** run here. diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py new file mode 100644 index 0000000..4fd8cd6 --- /dev/null +++ b/tests/unit/conftest.py @@ -0,0 +1,62 @@ +"""Unit-test helpers: load service modules without installing the full package tree.""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def load_module(module_name: str, relative_path: str): + """Load a .py file as a uniquely named module (avoids schemas.py collisions).""" + path = REPO_ROOT / relative_path + if module_name in sys.modules: + return sys.modules[module_name] + spec = importlib.util.spec_from_file_location(module_name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + # Service packages often import sibling modules by bare name. + sys.path.insert(0, str(path.parent)) + try: + spec.loader.exec_module(module) + finally: + if sys.path and sys.path[0] == str(path.parent): + sys.path.pop(0) + return module + + +@pytest.fixture(scope="session") +def embedding_schemas(): + return load_module( + "proseqgo_embedding_schemas", + "services/embedding-api/schemas.py", + ) + + +@pytest.fixture(scope="session") +def go_schemas(): + return load_module( + "proseqgo_go_schemas", + "services/go-prediction-api/schemas.py", + ) + + +@pytest.fixture(scope="session") +def embedding_validation(): + return load_module( + "proseqgo_embedding_validation", + "services/go-prediction-api/embedding_validation.py", + ) + + +@pytest.fixture(scope="session") +def ui_validation(): + return load_module( + "proseqgo_ui_validation", + "services/streamlit-ui/validation.py", + ) diff --git a/tests/unit/test_api_schemas.py b/tests/unit/test_api_schemas.py new file mode 100644 index 0000000..5e6c855 --- /dev/null +++ b/tests/unit/test_api_schemas.py @@ -0,0 +1,74 @@ +"""Tests for API request schema edge cases (Pydantic).""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + + +def test_create_job_request_valid(embedding_schemas) -> None: + req = embedding_schemas.CreateJobRequest( + sequences=[{"id": "p1", "sequence": "MKTAY"}], + ) + assert req.backend == "esm2" + assert req.batch_size == 8 + assert len(req.sequences) == 1 + + +def test_create_job_request_rejects_empty_sequences(embedding_schemas) -> None: + with pytest.raises(ValidationError): + embedding_schemas.CreateJobRequest(sequences=[]) + + +def test_create_job_request_rejects_empty_sequence_id(embedding_schemas) -> None: + with pytest.raises(ValidationError): + embedding_schemas.CreateJobRequest( + sequences=[{"id": "", "sequence": "ACDE"}], + ) + + +def test_create_job_request_batch_size_bounds(embedding_schemas) -> None: + with pytest.raises(ValidationError): + embedding_schemas.CreateJobRequest( + batch_size=0, + sequences=[{"id": "p1", "sequence": "ACDE"}], + ) + with pytest.raises(ValidationError): + embedding_schemas.CreateJobRequest( + batch_size=129, + sequences=[{"id": "p1", "sequence": "ACDE"}], + ) + + +def test_predict_go_from_sequences_timeout_bounds(embedding_schemas) -> None: + with pytest.raises(ValidationError): + embedding_schemas.PredictGoFromSequencesRequest( + sequences=[{"id": "p1", "sequence": "ACDE"}], + timeout_seconds=1, + ) + + +def test_job_status_literal(embedding_schemas) -> None: + ok = embedding_schemas.JobStatusResponse( + job_id="j1", + status="queued", + stage="test", + backend="esm2", + progress={"embedded_sequences": 0, "total_sequences": 1, "percent": 0.0}, + ) + assert ok.status == "queued" + with pytest.raises(ValidationError): + embedding_schemas.JobStatusResponse( + job_id="j1", + status="cancelled", + stage="test", + backend="esm2", + progress={}, + ) + + +def test_go_predict_request_top_k(go_schemas) -> None: + req = go_schemas.PredictRequest(embedding=[0.1] * 4, top_k=5) + assert req.top_k == 5 + with pytest.raises(ValidationError): + go_schemas.PredictRequest(embedding=[0.1], top_k=0) diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py new file mode 100644 index 0000000..3fe51c7 --- /dev/null +++ b/tests/unit/test_config.py @@ -0,0 +1,45 @@ +"""Tests for YAML config loading and validation.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +from src.config import Config, EMBEDDING_DIMS, load_config + + +def test_load_config_from_repo_yaml() -> None: + cfg = load_config("configs/config.yaml") + assert cfg.embedding_dim == EMBEDDING_DIMS["esm2"] + assert cfg.seed == 42 + assert cfg.num_labels == 500 + assert cfg.epochs == 60 + + +def test_config_unknown_embeddings_source() -> None: + with pytest.raises(ValueError, match="Unknown embeddings_source"): + Config(data={"embeddings_source": "not-a-backend"}) + + +def test_config_defaults_and_accessors(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PROJECT_ROOT", str(tmp_path)) + raw = { + "data": {"embeddings_source": "protbert", "num_labels": 10}, + "training": {}, + "output": {"output_dir": "out"}, + } + path = tmp_path / "cfg.yaml" + path.write_text(yaml.safe_dump(raw), encoding="utf-8") + + cfg = load_config(path) + assert cfg.embedding_dim == 1024 + assert cfg.batch_size == 128 + assert cfg.learning_rate == 1e-3 + assert cfg.output_dir == tmp_path / "out" + + +def test_load_config_missing_file(tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError): + load_config(tmp_path / "missing.yaml") diff --git a/tests/unit/test_embedding_validation.py b/tests/unit/test_embedding_validation.py new file mode 100644 index 0000000..6dbbf99 --- /dev/null +++ b/tests/unit/test_embedding_validation.py @@ -0,0 +1,23 @@ +"""Tests for GO prediction embedding vector validation.""" + +from __future__ import annotations + +import numpy as np +import pytest + + +def test_validate_embedding_accepts_list(embedding_validation) -> None: + out = embedding_validation.validate_embedding([0.1, 0.2, 0.3], expected_dim=3) + assert isinstance(out, np.ndarray) + assert out.dtype == np.float32 + assert out.shape == (3,) + + +def test_validate_embedding_wrong_dim(embedding_validation) -> None: + with pytest.raises(ValueError, match="length 4"): + embedding_validation.validate_embedding([1.0, 2.0], expected_dim=4) + + +def test_validate_embedding_rejects_2d(embedding_validation) -> None: + with pytest.raises(ValueError, match="1-dimensional"): + embedding_validation.validate_embedding([[1.0, 2.0]], expected_dim=2) diff --git a/tests/unit/test_sequences.py b/tests/unit/test_sequences.py new file mode 100644 index 0000000..7cca5b2 --- /dev/null +++ b/tests/unit/test_sequences.py @@ -0,0 +1,46 @@ +"""Tests for shared protein sequence helpers.""" + +from __future__ import annotations + +import pytest + +from src.preprocess.sequences import ( + extract_protein_id, + format_for_tokenizer, + normalize_sequence, +) + + +@pytest.mark.parametrize( + ("raw", "expected"), + [ + ("mktay", "MKTAY"), + (" MK TA\nY ", "MKTAY"), + ("ACDEFG", "ACDEFG"), + ("ACDEUB", "ACDEXX"), # U,B remapped + ("ACDE1Z", "ACDEXX"), # digit + Z remapped + ], +) +def test_normalize_sequence(raw: str, expected: str) -> None: + assert normalize_sequence(raw) == expected + + +@pytest.mark.parametrize( + ("header", "expected"), + [ + (">sp|P12345|GENE_HUMAN description", "P12345"), + (">tr|A0A000|NAME", "A0A000"), + (">SIMPLE_ID rest of header", "SIMPLE_ID"), + ("P99999 no_gt", "P99999"), + ], +) +def test_extract_protein_id(header: str, expected: str) -> None: + assert extract_protein_id(header) == expected + + +def test_format_for_tokenizer_space_separated() -> None: + assert format_for_tokenizer("ACDE", "space_separated") == "A C D E" + + +def test_format_for_tokenizer_passthrough() -> None: + assert format_for_tokenizer("ACDE", "esm") == "ACDE" diff --git a/tests/unit/test_ui_validation.py b/tests/unit/test_ui_validation.py new file mode 100644 index 0000000..87614ca --- /dev/null +++ b/tests/unit/test_ui_validation.py @@ -0,0 +1,59 @@ +"""Tests for Streamlit UI input validation (product gate before API calls).""" + +from __future__ import annotations + + +def test_ui_normalize_sequence(ui_validation) -> None: + assert ui_validation.normalize_sequence(" mk\nta ") == "MKTA" + + +def test_validate_sequence_ok(ui_validation) -> None: + ok, msg = ui_validation.validate_sequence("ACDEFG") + assert ok is True + assert msg == "" + + +def test_validate_sequence_empty(ui_validation) -> None: + ok, msg = ui_validation.validate_sequence("") + assert ok is False + assert "empty" in msg.lower() + + +def test_validate_sequence_rejects_noncanonical(ui_validation) -> None: + # UI is stricter than embedding normalize (no U/O remap at the gate). + ok, msg = ui_validation.validate_sequence("ACDEU") + assert ok is False + assert "invalid" in msg.lower() + + +def test_validate_fasta_upload_ok(ui_validation) -> None: + fasta = b">p1\nMKTAY\n>p2\nACDE\n" + ok, msg = ui_validation.validate_fasta_upload(fasta, "tiny.fasta") + assert ok is True + assert msg == "" + + +def test_validate_fasta_upload_no_records(ui_validation) -> None: + ok, msg = ui_validation.validate_fasta_upload(b"MKTAY\n", "bad.fasta") + assert ok is False + assert "no records" in msg.lower() + + +def test_validate_fasta_upload_empty(ui_validation) -> None: + ok, msg = ui_validation.validate_fasta_upload(b"", "empty.fasta") + assert ok is False + assert "empty" in msg.lower() + + +def test_validate_fasta_upload_too_large(ui_validation) -> None: + huge = b">p1\n" + (b"A" * (ui_validation.MAX_FASTA_UPLOAD_BYTES + 1)) + ok, msg = ui_validation.validate_fasta_upload(huge, "big.fasta") + assert ok is False + assert "exceeds" in msg.lower() + + +def test_validate_gateway_auth(ui_validation) -> None: + assert ui_validation.validate_gateway_auth("http://gw", "u", "p") is None + assert ui_validation.validate_gateway_auth(" ", "u", "p") is not None + assert ui_validation.validate_gateway_auth("http://gw", "", "p") is not None + assert ui_validation.validate_gateway_auth("http://gw", "u", "") is not None From 8db783ddc154396e2084c9703d5398065016d124 Mon Sep 17 00:00:00 2001 From: Behrouz Mirabdi Date: Fri, 24 Jul 2026 20:56:35 +0200 Subject: [PATCH 8/9] chore: remove obsolete go-term compose and cafa5 grafana dashboards Drop legacy docker_go_term compose files superseded by the root stack, and remove renamed cafa5-* Grafana JSON left after ProSeqGo dashboards. Co-authored-by: Cursor --- docker/docker_go_term/compose.api.yml | 43 -- docker/docker_go_term/docker-compose.yml | 120 ----- .../dashboards/cafa5-domain-pipelines.json | 494 ------------------ .../dashboards/cafa5-service-health.json | 196 ------- 4 files changed, 853 deletions(-) delete mode 100644 docker/docker_go_term/compose.api.yml delete mode 100644 docker/docker_go_term/docker-compose.yml delete mode 100644 monitoring/grafana/dashboards/cafa5-domain-pipelines.json delete mode 100644 monitoring/grafana/dashboards/cafa5-service-health.json diff --git a/docker/docker_go_term/compose.api.yml b/docker/docker_go_term/compose.api.yml deleted file mode 100644 index 895b70a..0000000 --- a/docker/docker_go_term/compose.api.yml +++ /dev/null @@ -1,43 +0,0 @@ -services: - embedding-api: - build: - context: ../.. - dockerfile: docker/docker_embedding/Dockerfile.embedding-api - image: cafa5-embedding-api:local - working_dir: /app/services/embedding-api - environment: - PYTHONUNBUFFERED: "1" - PYTHONPATH: /app:/app/services/embedding-api - GO_PREDICTION_API_URL: http://go-prediction-api:8000 - MLFLOW_TRACKING_URI: http://mlflow:5000 - depends_on: - - go-prediction-api - - mlflow - volumes: - - ../../outputs:/app/outputs - - ../../data/hf_cache:/app/data/hf_cache - ports: - - "8000:8000" - - go-prediction-api: - build: - context: ../.. - dockerfile: docker/docker_go_term/Dockerfile.api - image: cafa5-go-prediction-api:local - working_dir: /app/services/go-prediction-api - environment: - PYTHONUNBUFFERED: "1" - PYTHONPATH: /app:/app/services/go-prediction-api - MLFLOW_TRACKING_URI: http://mlflow:5000 - depends_on: - - mlflow - ports: - - "8001:8000" - - mlflow: - image: ghcr.io/mlflow/mlflow:v2.13.0 - command: mlflow server --host 0.0.0.0 --port 5000 - volumes: - - ../../mlruns:/mlruns - ports: - - "5000:5000" diff --git a/docker/docker_go_term/docker-compose.yml b/docker/docker_go_term/docker-compose.yml deleted file mode 100644 index b17fe3c..0000000 --- a/docker/docker_go_term/docker-compose.yml +++ /dev/null @@ -1,120 +0,0 @@ -services: - cafa5-cpu: - profiles: ["cpu"] - build: - context: . - dockerfile: Dockerfile - target: runtime-cpu - image: cafa5-runner:cpu-dev - working_dir: /app - stdin_open: true - tty: true - environment: - PYTHONUNBUFFERED: "1" - PYTHONPATH: /app - MLFLOW_TRACKING_URI: ${MLFLOW_TRACKING_URI:-file:/mlruns} - volumes: - - ./src:/app/src:z - - ./scripts:/app/scripts:z - - ./services:/app/services:z - - ./configs:/app/configs:ro,z - - ./data:/app/data:z - - ./outputs:/app/outputs:z - - ./mlruns:/mlruns:z - - ./README.md:/app/README.md:ro,z - - ./pyproject.toml:/app/pyproject.toml:ro,z - command: bash - - cafa5-nvidia: - profiles: ["nvidia"] - build: - context: . - dockerfile: Dockerfile - target: runtime-nvidia - image: cafa5-runner:nvidia-dev - working_dir: /app - stdin_open: true - tty: true - environment: - PYTHONUNBUFFERED: "1" - PYTHONPATH: /app - MLFLOW_TRACKING_URI: ${MLFLOW_TRACKING_URI:-file:/mlruns} - volumes: - - ./src:/app/src:z - - ./scripts:/app/scripts:z - - ./services:/app/services:z - - ./configs:/app/configs:ro,z - - ./data:/app/data:z - - ./outputs:/app/outputs:z - - ./mlruns:/mlruns:z - - ./README.md:/app/README.md:ro,z - - ./pyproject.toml:/app/pyproject.toml:ro,z - gpus: all - command: bash - - cafa5-amd: - profiles: ["amd"] - build: - context: . - dockerfile: Dockerfile - target: runtime-amd - args: - ROCM_BASE_IMAGE: ${ROCM_BASE_IMAGE:-rocm/pytorch:latest} - image: cafa5-runner:amd-dev - working_dir: /app - stdin_open: true - tty: true - environment: - PYTHONUNBUFFERED: "1" - PYTHONPATH: /app - MLFLOW_TRACKING_URI: ${MLFLOW_TRACKING_URI:-file:/mlruns} - HSA_OVERRIDE_GFX_VERSION: "12.0.0" - HIP_VISIBLE_DEVICES: "0" - ROCR_VISIBLE_DEVICES: "0" - volumes: - - ./src:/app/src:z - - ./scripts:/app/scripts:z - - ./services:/app/services:z - - ./configs:/app/configs:ro,z - - ./data:/app/data:z - - ./outputs:/app/outputs:z - - ./mlruns:/mlruns:z - - ./README.md:/app/README.md:ro,z - - ./pyproject.toml:/app/pyproject.toml:ro,z - devices: - - /dev/kfd - - /dev/dri - group_add: - - video - cap_add: - - SYS_PTRACE - security_opt: - - seccomp=unconfined - ipc: host - shm_size: "8gb" - command: bash - - embedding-api: - profiles: ["api"] - build: - context: . - dockerfile: Dockerfile - target: runtime-api - image: cafa5-embedding-api:dev - working_dir: /app/services/embedding-api - stdin_open: true - tty: true - environment: - PYTHONUNBUFFERED: "1" - PYTHONPATH: /app:/app/services/embedding-api - volumes: - - ./src:/app/src:z - - ./services:/app/services:z - - ./configs:/app/configs:ro,z - - ./data:/app/data:z - - ./outputs:/app/outputs:z - - ./README.md:/app/README.md:ro,z - - ./pyproject.toml:/app/pyproject.toml:ro,z - ports: - - "8000:8000" - command: uvicorn main:app --host 0.0.0.0 --port 8000 --reload diff --git a/monitoring/grafana/dashboards/cafa5-domain-pipelines.json b/monitoring/grafana/dashboards/cafa5-domain-pipelines.json deleted file mode 100644 index 798a300..0000000 --- a/monitoring/grafana/dashboards/cafa5-domain-pipelines.json +++ /dev/null @@ -1,494 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [], - "liveNow": false, - "panels": [ - { - "id": 1, - "type": "text", - "title": "Embedding", - "gridPos": { "h": 2, "w": 24, "x": 0, "y": 0 }, - "options": { - "content": "## Embedding Pipeline", - "mode": "markdown" - } - }, - { - "id": 2, - "type": "timeseries", - "title": "Embedding Queue Depth by Status", - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 2 }, - "targets": [ - { - "expr": "sum by (status) (cafa5_embedding_queue_jobs)", - "legendFormat": "{{status}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { "unit": "none", "min": 0 }, - "overrides": [] - }, - "options": { - "legend": { "displayMode": "list", "placement": "bottom" } - } - }, - { - "id": 3, - "type": "barchart", - "title": "Embedding Job Outcomes (15m)", - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 2 }, - "targets": [ - { - "expr": "sum by (status, backend) (increase(cafa5_embedding_jobs_total{backend=~\"$backend\"}[15m]))", - "legendFormat": "{{backend}} {{status}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { "unit": "none", "min": 0 }, - "overrides": [] - }, - "options": { - "legend": { "displayMode": "table", "placement": "bottom" } - } - }, - { - "id": 4, - "type": "timeseries", - "title": "Embedding Job Duration P95 (s)", - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 2 }, - "targets": [ - { - "expr": "histogram_quantile(0.95, sum by (le, status, backend) (rate(cafa5_embedding_job_duration_seconds_bucket{backend=~\"$backend\"}[5m])))", - "legendFormat": "{{backend}} {{status}} p95", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { "unit": "s", "min": 0 }, - "overrides": [] - }, - "options": { - "legend": { "displayMode": "table", "placement": "bottom" } - } - }, - { - "id": 5, - "type": "timeseries", - "title": "Sequence Length Distribution (rate by bucket)", - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 10 }, - "targets": [ - { - "expr": "sum by (le, backend) (rate(cafa5_embedding_sequence_length_bucket{backend=~\"$backend\"}[5m]))", - "legendFormat": "{{backend}} <= {{le}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { "unit": "none", "min": 0 }, - "overrides": [] - }, - "options": { - "legend": { "displayMode": "table", "placement": "bottom" } - } - }, - { - "id": 6, - "type": "timeseries", - "title": "Embedding Sequence Throughput (seq/s)", - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 10 }, - "targets": [ - { - "expr": "sum by (backend) (rate(cafa5_embedding_sequences_processed_total{backend=~\"$backend\"}[5m]))", - "legendFormat": "{{backend}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { "unit": "reqps", "min": 0 }, - "overrides": [] - }, - "options": { - "legend": { "displayMode": "list", "placement": "bottom" } - } - }, - { - "id": 18, - "type": "timeseries", - "title": "RQ Queue Length (Redis)", - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 18 }, - "targets": [ - { - "expr": "cafa5_rq_queue_length", - "legendFormat": "{{queue}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { "unit": "none", "min": 0 }, - "overrides": [] - }, - "options": { - "legend": { "displayMode": "list", "placement": "bottom" } - } - }, - { - "id": 19, - "type": "timeseries", - "title": "Redis Memory Used", - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 18 }, - "targets": [ - { - "expr": "redis_memory_used_bytes", - "legendFormat": "used", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { "unit": "bytes", "min": 0 }, - "overrides": [] - }, - "options": { - "legend": { "displayMode": "list", "placement": "bottom" } - } - }, - - { - "id": 7, - "type": "text", - "title": "Training", - "gridPos": { "h": 2, "w": 24, "x": 0, "y": 26 }, - "options": { - "content": "## Training Pipeline", - "mode": "markdown" - } - }, - { - "id": 8, - "type": "timeseries", - "title": "Training Queue Depth by Status", - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 28 }, - "targets": [ - { - "expr": "sum by (status) (cafa5_training_queue_jobs)", - "legendFormat": "{{status}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { "unit": "none", "min": 0 }, - "overrides": [] - }, - "options": { - "legend": { "displayMode": "list", "placement": "bottom" } - } - }, - { - "id": 9, - "type": "barchart", - "title": "Training Failure Reasons (15m)", - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 28 }, - "targets": [ - { - "expr": "sum by (reason) (increase(cafa5_training_subprocess_failures_total{reason=~\"$reason\"}[15m]))", - "legendFormat": "{{reason}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { "unit": "none", "min": 0 }, - "overrides": [] - }, - "options": { - "legend": { "displayMode": "table", "placement": "bottom" } - } - }, - { - "id": 10, - "type": "timeseries", - "title": "Training Duration P95 by Mode/Status (s)", - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 28 }, - "targets": [ - { - "expr": "histogram_quantile(0.95, sum by (le, mode, status) (rate(cafa5_training_job_duration_seconds_bucket{mode=~\"$mode\"}[5m])))", - "legendFormat": "{{mode}} {{status}} p95", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { "unit": "s", "min": 0 }, - "overrides": [] - }, - "options": { - "legend": { "displayMode": "table", "placement": "bottom" } - } - }, - { - "id": 11, - "type": "timeseries", - "title": "Training Job Outcomes by Mode (15m)", - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 36 }, - "targets": [ - { - "expr": "sum by (mode, status) (increase(cafa5_training_jobs_total{mode=~\"$mode\"}[15m]))", - "legendFormat": "{{mode}} {{status}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { "unit": "none", "min": 0 }, - "overrides": [] - }, - "options": { - "legend": { "displayMode": "table", "placement": "bottom" } - } - }, - - { - "id": 12, - "type": "text", - "title": "GO Inference", - "gridPos": { "h": 2, "w": 24, "x": 0, "y": 44 }, - "options": { - "content": "## GO Inference", - "mode": "markdown" - } - }, - { - "id": 13, - "type": "timeseries", - "title": "Inference Latency P95 by Model Version (s)", - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 46 }, - "targets": [ - { - "expr": "histogram_quantile(0.95, sum by (le, model_version) (rate(cafa5_inference_duration_seconds_bucket{model_version=~\"$model_version\"}[5m])))", - "legendFormat": "{{model_version}} p95", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { "unit": "s", "min": 0 }, - "overrides": [] - }, - "options": { - "legend": { "displayMode": "list", "placement": "bottom" } - } - }, - { - "id": 14, - "type": "timeseries", - "title": "Inference Request Rate by Model/Status", - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 46 }, - "targets": [ - { - "expr": "sum by (model_version, status_code) (rate(cafa5_inference_requests_total{model_version=~\"$model_version\"}[5m]))", - "legendFormat": "{{model_version}} {{status_code}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { "unit": "reqps", "min": 0 }, - "overrides": [] - }, - "options": { - "legend": { "displayMode": "table", "placement": "bottom" } - } - }, - { - "id": 15, - "type": "barchart", - "title": "Input Validation Failures by Reason (15m)", - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 46 }, - "targets": [ - { - "expr": "sum by (reason) (increase(cafa5_inference_input_validation_failures_total{reason=~\"$reason\"}[15m]))", - "legendFormat": "{{reason}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { "unit": "none", "min": 0 }, - "overrides": [] - }, - "options": { - "legend": { "displayMode": "table", "placement": "bottom" } - } - }, - { - "id": 16, - "type": "barchart", - "title": "Top-K Usage (1h)", - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 54 }, - "targets": [ - { - "expr": "sum by (top_k) (increase(cafa5_inference_top_k_requests_total{top_k=~\"$top_k\"}[1h]))", - "legendFormat": "top_k={{top_k}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { "unit": "none", "min": 0 }, - "overrides": [] - }, - "options": { - "legend": { "displayMode": "list", "placement": "bottom" } - } - }, - { - "id": 17, - "type": "timeseries", - "title": "Inference Error Ratio (5m)", - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 54 }, - "targets": [ - { - "expr": "sum(rate(cafa5_inference_requests_total{model_version=~\"$model_version\",status_code=~\"5..\"}[5m])) / clamp_min(sum(rate(cafa5_inference_requests_total{model_version=~\"$model_version\"}[5m])), 1e-9)", - "legendFormat": "5xx ratio", - "refId": "A" - }, - { - "expr": "sum(rate(cafa5_inference_requests_total{model_version=~\"$model_version\",status_code=~\"4..\"}[5m])) / clamp_min(sum(rate(cafa5_inference_requests_total{model_version=~\"$model_version\"}[5m])), 1e-9)", - "legendFormat": "4xx ratio", - "refId": "B" - } - ], - "fieldConfig": { - "defaults": { "unit": "percentunit", "min": 0, "max": 1 }, - "overrides": [] - }, - "options": { - "legend": { "displayMode": "list", "placement": "bottom" } - } - } - ], - "refresh": "30s", - "schemaVersion": 39, - "style": "dark", - "tags": ["cafa5", "domain", "mlops", "embedding", "training", "inference"], - "templating": { - "list": [ - { - "name": "backend", - "type": "query", - "label": "Embedding Backend", - "hide": 0, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "definition": "label_values(cafa5_embedding_jobs_total, backend)", - "query": { - "query": "label_values(cafa5_embedding_jobs_total, backend)", - "refId": "BackendVar" - }, - "multi": true, - "includeAll": true, - "allValue": ".*", - "current": { "text": "All", "value": ".*" } - }, - { - "name": "mode", - "type": "query", - "label": "Training Mode", - "hide": 0, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "definition": "label_values(cafa5_training_jobs_total, mode)", - "query": { - "query": "label_values(cafa5_training_jobs_total, mode)", - "refId": "ModeVar" - }, - "multi": true, - "includeAll": true, - "allValue": ".*", - "current": { "text": "All", "value": ".*" } - }, - { - "name": "model_version", - "type": "query", - "label": "Model Version", - "hide": 0, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "definition": "label_values(cafa5_inference_requests_total, model_version)", - "query": { - "query": "label_values(cafa5_inference_requests_total, model_version)", - "refId": "ModelVerVar" - }, - "multi": true, - "includeAll": true, - "allValue": ".*", - "current": { "text": "All", "value": ".*" } - }, - { - "name": "reason", - "type": "query", - "label": "Failure Reason", - "hide": 0, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "definition": "label_values(cafa5_training_subprocess_failures_total, reason)", - "query": { - "query": "label_values(cafa5_training_subprocess_failures_total, reason)", - "refId": "ReasonVar" - }, - "multi": true, - "includeAll": true, - "allValue": ".*", - "current": { "text": "All", "value": ".*" } - }, - { - "name": "top_k", - "type": "query", - "label": "Top-K", - "hide": 0, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "definition": "label_values(cafa5_inference_top_k_requests_total, top_k)", - "query": { - "query": "label_values(cafa5_inference_top_k_requests_total, top_k)", - "refId": "TopKVar" - }, - "multi": true, - "includeAll": true, - "allValue": ".*", - "current": { "text": "All", "value": ".*" } - } - ] - }, - "time": { "from": "now-6h", "to": "now" }, - "timepicker": {}, - "timezone": "", - "title": "CAFA5 Domain Pipelines", - "uid": "cafa5-domain-pipelines", - "version": 1, - "weekStart": "" - } \ No newline at end of file diff --git a/monitoring/grafana/dashboards/cafa5-service-health.json b/monitoring/grafana/dashboards/cafa5-service-health.json deleted file mode 100644 index 8d78840..0000000 --- a/monitoring/grafana/dashboards/cafa5-service-health.json +++ /dev/null @@ -1,196 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [], - "liveNow": false, - "panels": [ - { - "id": 1, - "type": "stat", - "title": "Targets Up (metrics jobs)", - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "gridPos": { "h": 6, "w": 6, "x": 0, "y": 0 }, - "targets": [ - { - "expr": "sum(up{job=~\".*_metrics\"})", - "refId": "A" - } - ], - "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "value" - }, - "fieldConfig": { - "defaults": { - "unit": "none", - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "red", "value": null }, - { "color": "green", "value": 1 } - ] - } - }, - "overrides": [] - } - }, - { - "id": 2, - "type": "timeseries", - "title": "Request Rate (req/s)", - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "gridPos": { "h": 8, "w": 18, "x": 6, "y": 0 }, - "targets": [ - { - "expr": "sum by (service, route) (rate(cafa5_http_requests_total{service=~\"$service\",route=~\"$route\",status_code=~\"$status_code\"}[5m]))", - "legendFormat": "{{service}} {{route}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { "unit": "reqps" }, - "overrides": [] - }, - "options": { "legend": { "displayMode": "table", "placement": "bottom" } } - }, - { - "id": 3, - "type": "timeseries", - "title": "Error Ratio (5xx / total)", - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 6 }, - "targets": [ - { - "expr": "sum(rate(cafa5_http_requests_total{service=~\"$service\",route=~\"$route\",status_code=~\"5..\"}[5m])) / clamp_min(sum(rate(cafa5_http_requests_total{service=~\"$service\",route=~\"$route\"}[5m])), 1e-9)", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { "unit": "percentunit", "min": 0, "max": 1 }, - "overrides": [] - }, - "options": { "legend": { "displayMode": "list", "placement": "bottom" } } - }, - { - "id": 4, - "type": "timeseries", - "title": "P95 Latency (s)", - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 6 }, - "targets": [ - { - "expr": "histogram_quantile(0.95, sum by (le, service, route) (rate(cafa5_http_request_duration_seconds_bucket{service=~\"$service\",route=~\"$route\",status_code=~\"$status_code\"}[5m])))", - "legendFormat": "{{service}} {{route}} p95", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { "unit": "s" }, - "overrides": [] - }, - "options": { "legend": { "displayMode": "table", "placement": "bottom" } } - }, - { - "id": 5, - "type": "timeseries", - "title": "In-flight Requests", - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 6 }, - "targets": [ - { - "expr": "sum by (service) (cafa5_http_in_flight_requests{service=~\"$service\"})", - "legendFormat": "{{service}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { "unit": "none", "min": 0 }, - "overrides": [] - }, - "options": { "legend": { "displayMode": "list", "placement": "bottom" } } - } - ], - "refresh": "30s", - "schemaVersion": 39, - "style": "dark", - "tags": ["cafa5", "operations", "http"], - "templating": { - "list": [ - { - "name": "service", - "type": "query", - "label": "Service", - "hide": 0, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "definition": "label_values(cafa5_http_requests_total, service)", - "query": { - "query": "label_values(cafa5_http_requests_total, service)", - "refId": "ServiceVar" - }, - "multi": true, - "includeAll": true, - "allValue": ".*", - "current": { "text": "All", "value": ".*" } - }, - { - "name": "route", - "type": "query", - "label": "Route", - "hide": 0, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "definition": "label_values(cafa5_http_requests_total{service=~\"$service\"}, route)", - "query": { - "query": "label_values(cafa5_http_requests_total{service=~\"$service\"}, route)", - "refId": "RouteVar" - }, - "multi": true, - "includeAll": true, - "allValue": ".*", - "current": { "text": "All", "value": ".*" } - }, - { - "name": "status_code", - "type": "query", - "label": "Status Code", - "hide": 0, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "definition": "label_values(cafa5_http_requests_total{service=~\"$service\",route=~\"$route\"}, status_code)", - "query": { - "query": "label_values(cafa5_http_requests_total{service=~\"$service\",route=~\"$route\"}, status_code)", - "refId": "StatusVar" - }, - "multi": true, - "includeAll": true, - "allValue": ".*", - "current": { "text": "All", "value": ".*" } - } - ] - }, - "time": { "from": "now-6h", "to": "now" }, - "timepicker": {}, - "timezone": "", - "title": "CAFA5 Service Health", - "uid": "cafa5-service-health", - "version": 1, - "weekStart": "" - } \ No newline at end of file From a45f9469d561e0318fa90f2cbdfcb70b76a74b22 Mon Sep 17 00:00:00 2001 From: Behrouz Mirabdi Date: Fri, 24 Jul 2026 21:04:08 +0200 Subject: [PATCH 9/9] ci: publish product images to GHCR on main Push immutable sha- and moving main tags for all five images on main merges; keep PRs build-only. Add make pull-images for local pulls. Co-authored-by: Cursor --- .github/CI.md | 35 +++++++++++++++++------------- .github/workflows/ci.yml | 46 ++++++++++++++++++++++++++++++++++++++-- Makefile | 20 ++++++++++++++++- README.md | 7 +++--- 4 files changed, 87 insertions(+), 21 deletions(-) diff --git a/.github/CI.md b/.github/CI.md index 930c90d..24022e4 100644 --- a/.github/CI.md +++ b/.github/CI.md @@ -6,27 +6,32 @@ |---------|-----|---------| | `pull_request`, `push` to `main` | Lint | `make lint` → `ruff check src services scripts` | | `pull_request`, `push` to `main` | Unit tests (after lint) | `make test` → `pytest tests/unit -q` | -| `pull_request`, `push` to `main` | Image builds (after lint, parallel) | Buildx build of all 5 product images (**no push**) | +| `pull_request` | Image builds (after lint, parallel) | Buildx build of all 5 product images (**no push**) | +| `push` to `main` | Image builds + GHCR publish | Build and push `sha-` + `main` tags | Unit tests are **Docker/GPU/network free**. They cover sequence normalization, config loading, API schema bounds, embedding vector validation, and UI input gates. -### Image builds (Phase 1C) +### Image builds -| Image | Dockerfile | -|-------|------------| -| `proseqgo-embedding-api` | `docker/docker_embedding/Dockerfile.embedding-api` | -| `proseqgo-go-prediction-api` | `docker/docker_go_term/Dockerfile.api` | -| `proseqgo-streamlit-ui` | `docker/docker_streamlit/Dockerfile.streamlit` | -| `proseqgo-trainer-api` | `docker/docker_training/Dockerfile.training` | -| `proseqgo-mlflow` | `docker/docker_mlflow/Dockerfile` | +| Image | Dockerfile | GHCR repository | +|-------|------------|-----------------| +| `proseqgo-embedding-api` | `docker/docker_embedding/Dockerfile.embedding-api` | `ghcr.io/behroooz/proseqgo-embedding-api` | +| `proseqgo-go-prediction-api` | `docker/docker_go_term/Dockerfile.api` | `ghcr.io/behroooz/proseqgo-go-prediction-api` | +| `proseqgo-streamlit-ui` | `docker/docker_streamlit/Dockerfile.streamlit` | `ghcr.io/behroooz/proseqgo-streamlit-ui` | +| `proseqgo-trainer-api` | `docker/docker_training/Dockerfile.training` | `ghcr.io/behroooz/proseqgo-trainer-api` | +| `proseqgo-mlflow` | `docker/docker_mlflow/Dockerfile` | `ghcr.io/behroooz/proseqgo-mlflow` | -- PR/`main`: **build only** (GHA layer cache, `outputs: type=cacheonly`) -- Torch images use **CPU wheels** in CI (`TORCH_INDEX_URL=.../cpu`) for smaller/faster builds; local Compose still defaults to CUDA index -- Local: `make build-images` (or `TORCH_INDEX_URL=https://download.pytorch.org/whl/cpu make build-images`) +Policy: -## Registry (Phase 2) +```text +PR → build only (GHA layer cache, no push) +main → build + push sha- and moving main +``` -Images will publish to **GHCR** (`ghcr.io/BehRoooz/...`) with immutable `sha-` tags. Not wired yet. +- Torch images use **CPU wheels** in CI (`TORCH_INDEX_URL=.../cpu`) for smaller/faster builds; local Compose still defaults to CUDA index +- Local build: `make build-images` +- Pull published images: `make pull-images` or `GHCR_TAG=sha- make pull-images` +- First publish happens after this workflow runs on **`main`**. Packages may be private by default; set package visibility in GitHub Packages if others need to pull. ## Compose / secrets in CI (Phase 3) @@ -42,4 +47,4 @@ Use throwaway passwords from `.env.example` only inside ephemeral CI runners. - No training / GPU / full retrain jobs in PR or `main` CI - No Training API profile in automated smoke (serving stack only) -- No registry push yet (Phase 2); Compose smoke lands in Phase 3 +- Compose smoke lands in Phase 3 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 34d16af..3a475ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,11 @@ concurrency: permissions: contents: read actions: write + packages: write + +env: + # GHCR requires lowercase owner/image names. + REGISTRY: ghcr.io jobs: lint: @@ -53,7 +58,7 @@ jobs: - name: Run unit tests run: make test - # Phase 1C: prove all product images build. No registry push (Phase 2). + # Phase 1C/2: build all product images; push to GHCR only on main. build-images: name: Build ${{ matrix.name }} runs-on: ubuntu-latest @@ -93,17 +98,54 @@ jobs: /usr/local/.ghcup || true df -h + - name: Set image name and push flag + id: meta + run: | + owner="$(echo '${{ github.repository_owner }}' | tr '[:upper:]' '[:lower:]')" + echo "image=${{ env.REGISTRY }}/${owner}/${{ matrix.image }}" >> "$GITHUB_OUTPUT" + if [ '${{ github.event_name }}' = 'push' ] && [ '${{ github.ref }}' = 'refs/heads/main' ]; then + echo "push=true" >> "$GITHUB_OUTPUT" + else + echo "push=false" >> "$GITHUB_OUTPUT" + fi + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Log in to GHCR + if: steps.meta.outputs.push == 'true' + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # PR / non-main: prove the image builds; do not push. - name: Build image (no push) + if: steps.meta.outputs.push != 'true' uses: docker/build-push-action@v6 with: context: . file: ${{ matrix.dockerfile }} push: false outputs: type=cacheonly - tags: ${{ matrix.image }}:sha-${{ github.sha }} + tags: ${{ steps.meta.outputs.image }}:sha-${{ github.sha }} + build-args: ${{ matrix.build_args }} + cache-from: type=gha,scope=build-${{ matrix.name }} + cache-to: type=gha,mode=max,scope=build-${{ matrix.name }} + + # main: publish immutable sha tag + moving main tag. + - name: Build and push image + if: steps.meta.outputs.push == 'true' + uses: docker/build-push-action@v6 + with: + context: . + file: ${{ matrix.dockerfile }} + push: true + tags: | + ${{ steps.meta.outputs.image }}:sha-${{ github.sha }} + ${{ steps.meta.outputs.image }}:main build-args: ${{ matrix.build_args }} cache-from: type=gha,scope=build-${{ matrix.name }} cache-to: type=gha,mode=max,scope=build-${{ matrix.name }} + provenance: false diff --git a/Makefile b/Makefile index 29b5d3c..b7ed0a0 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .PHONY: help up down restart training-up training-down monitoring-up monitoring-down \ - up-all down-all lint test build-images smoke ci-env + up-all down-all lint test build-images pull-images smoke ci-env # Product Python paths linted in CI (Phase 1A). Expand later if needed. LINT_PATHS := src services scripts @@ -14,6 +14,16 @@ MLFLOW_IMAGE ?= proseqgo-mlflow:local # Local default matches GPU Compose; CI passes cpu index via TORCH_INDEX_URL. TORCH_INDEX_URL ?= https://download.pytorch.org/whl/cu132 +# GHCR (Phase 2). Owner must be lowercase. Tag: main | sha- +GHCR_OWNER ?= behroooz +GHCR_REGISTRY ?= ghcr.io +GHCR_TAG ?= main +GHCR_EMBEDDING_IMAGE ?= $(GHCR_REGISTRY)/$(GHCR_OWNER)/proseqgo-embedding-api:$(GHCR_TAG) +GHCR_GO_PRED_IMAGE ?= $(GHCR_REGISTRY)/$(GHCR_OWNER)/proseqgo-go-prediction-api:$(GHCR_TAG) +GHCR_STREAMLIT_IMAGE ?= $(GHCR_REGISTRY)/$(GHCR_OWNER)/proseqgo-streamlit-ui:$(GHCR_TAG) +GHCR_TRAINER_IMAGE ?= $(GHCR_REGISTRY)/$(GHCR_OWNER)/proseqgo-trainer-api:$(GHCR_TAG) +GHCR_MLFLOW_IMAGE ?= $(GHCR_REGISTRY)/$(GHCR_OWNER)/proseqgo-mlflow:$(GHCR_TAG) + help: @echo "Available targets:" @echo " make up - Start default services with Docker Compose" @@ -25,6 +35,7 @@ help: @echo " make lint - Ruff check on src/ services/ scripts/" @echo " make test - Unit tests (tests/unit; Phase 1B)" @echo " make build-images - Build all product Docker images" + @echo " make pull-images - Pull product images from GHCR (GHCR_TAG=main|sha-...)" @echo " make smoke - Run Compose smoke scripts (stack must be up)" @echo " make ci-env - Copy .env.example -> .env for local/CI Compose" @@ -89,6 +100,13 @@ build-images: -t $(TRAINER_IMAGE) . docker build -f docker/docker_mlflow/Dockerfile -t $(MLFLOW_IMAGE) . +pull-images: + docker pull $(GHCR_EMBEDDING_IMAGE) + docker pull $(GHCR_GO_PRED_IMAGE) + docker pull $(GHCR_STREAMLIT_IMAGE) + docker pull $(GHCR_TRAINER_IMAGE) + docker pull $(GHCR_MLFLOW_IMAGE) + # Expects default Compose stack already healthy. Does not start training/GPU jobs. smoke: ./tests/smoke/smoke_embedding_api.sh diff --git a/README.md b/README.md index eb501a7..487c44d 100644 --- a/README.md +++ b/README.md @@ -466,18 +466,19 @@ make monitoring-down make lint # Ruff on src/ services/ scripts/ make test # Unit tests (tests/unit) make build-images # Build all five product images +make pull-images # Pull product images from GHCR (GHCR_TAG=main|sha-...) make smoke # Smoke scripts (Compose stack must already be up) make ci-env # Copy .env.example → .env if missing ``` ## CI (GitHub Actions) -PR and `main` pushes run **lint**, **unit tests**, and **parallel image builds** (no registry push yet). See [`.github/CI.md`](.github/CI.md). +PR and `main` pushes run **lint**, **unit tests**, and **parallel image builds**. Merges to `main` also **publish to GHCR** (`sha-` + `main`). See [`.github/CI.md`](.github/CI.md). -- Registry target for later phases: **GHCR** +- Registry: **GHCR** (`ghcr.io/behroooz/proseqgo-*`) - CI does **not** run training/GPU/retrain jobs - Compose in CI will use `make ci-env` (`.env.example` only)—never commit real secrets -- Local image rebuild: `make build-images` +- Local image rebuild: `make build-images`; pull published: `make pull-images` ## Service-Specific Documentation