docker-compose.yml

# Integrated stack: Embedding API + GO Prediction API + MLflow + Training API.
# Single public entry: nginx on port 80. Portable base (CPU-safe; no gpus: here).
#
# Local (recommended):
#   make up              # auto-adds docker-compose.gpu.yml when NVIDIA is available
#   make training-up     # --profile training
#   make monitoring-up   # --profile monitoring
#
# Manual compose:
#   docker compose up --build
#   docker compose -f docker-compose.yml -f docker-compose.gpu.yml up --build   # GPU
#   docker compose -f docker-compose.yml -f docker-compose.ci.yml up --build    # CI / CPU smoke
#
# Copy .env.example to .env and set secrets before first run.
networks:
  proseqgo:
    driver: bridge

volumes:
  prometheus_data:
  grafana_data:
  postgres_data:
  minio_data:
  redis_data:

services:
  postgres:
    image: postgres:16-alpine
    networks: [proseqgo]
    environment:
      POSTGRES_USER: ${POSTGRES_USER}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
      POSTGRES_DB: ${POSTGRES_DB}
    volumes:
      - postgres_data:/var/lib/postgresql/data
      - ./docker/postgres/init-proseqgo-jobs.sh:/docker-entrypoint-initdb.d/01-proseqgo-jobs.sh:ro
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"]
      interval: 5s
      timeout: 5s
      retries: 10
      start_period: 10s
    restart: unless-stopped

  redis:
    image: redis:7-alpine
    networks: [proseqgo]
    command: ["redis-server", "--appendonly", "yes"]
    volumes:
      - redis_data:/data
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 5s
      timeout: 3s
      retries: 10
      start_period: 5s
    restart: unless-stopped

  minio:
    image: minio/minio:latest
    networks: [proseqgo]
    command: server /data --console-address ":9001"
    environment:
      MINIO_ROOT_USER: ${MINIO_ROOT_USER}
      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
    volumes:
      - minio_data:/data
    ports:
      - "127.0.0.1:9000:9000"
      - "127.0.0.1:9001:9001"
    healthcheck:
      test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:9000/minio/health/live || exit 1"]
      interval: 5s
      timeout: 5s
      retries: 10
      start_period: 10s
    restart: unless-stopped

  minio-init:
    image: minio/mc:latest
    networks: [proseqgo]
    depends_on:
      minio:
        condition: service_healthy
    environment:
      MINIO_ROOT_USER: ${MINIO_ROOT_USER}
      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
    entrypoint: >
      /bin/sh -c "
      mc alias set local http://minio:9000 $$MINIO_ROOT_USER $$MINIO_ROOT_PASSWORD &&
      mc mb --ignore-existing local/mlflow-artifacts &&
      mc mb --ignore-existing local/mlflow-db-backups &&
      echo 'MinIO buckets ready'
      "
    restart: "no"

  mlflow:
    build:
      context: .
      dockerfile: docker/docker_mlflow/Dockerfile
    image: proseqgo-mlflow:local
    networks: [proseqgo]
    depends_on:
      postgres:
        condition: service_healthy
      minio-init:
        condition: service_completed_successfully
    environment:
      MLFLOW_S3_ENDPOINT_URL: ${MLFLOW_S3_ENDPOINT_URL}
      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
      AWS_DEFAULT_REGION: us-east-1
      MLFLOW_S3_IGNORE_TLS: "true"
    command: >
      mlflow server
      --host 0.0.0.0 --port 5000
      --backend-store-uri postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB}
      --default-artifact-root ${MLFLOW_ARTIFACT_ROOT}
      --artifacts-destination ${MLFLOW_ARTIFACT_ROOT}
      --serve-artifacts
    restart: unless-stopped

  postgres-backup:
    image: prodrigestivill/postgres-backup-local:16
    networks: [proseqgo]
    depends_on:
      postgres:
        condition: service_healthy
    environment:
      POSTGRES_HOST: postgres
      POSTGRES_PORT: 5432
      POSTGRES_DB: ${POSTGRES_DB}
      POSTGRES_USER: ${POSTGRES_USER}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
      SCHEDULE: "@daily"
      BACKUP_KEEP_DAYS: ${BACKUP_RETENTION_DAYS}
      BACKUP_KEEP_WEEKS: ${BACKUP_RETENTION_WEEKS}
      BACKUP_DIR: /backups
    volumes:
      - ./backups/postgres:/backups
    restart: unless-stopped

  backup-offload:
    image: minio/mc:latest
    networks: [proseqgo]
    depends_on:
      minio:
        condition: service_healthy
      postgres-backup:
        condition: service_started
    environment:
      MLFLOW_S3_ENDPOINT_URL: ${MLFLOW_S3_ENDPOINT_URL}
      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
      BACKUP_OFFLOAD_TARGET: ${BACKUP_OFFLOAD_TARGET}
      BACKUP_DIR: /backups
      BACKUP_OFFLOAD_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS}
    volumes:
      - ./backups/postgres:/backups:ro
      - ./scripts/offload_postgres_backup.sh:/offload.sh:ro
    entrypoint: /bin/sh
    command: -c "while true; do /offload.sh; sleep 3600; done"
    restart: unless-stopped

  trainer-api:
    build:
      context: .
      dockerfile: docker/docker_training/Dockerfile.training
    image: proseqgo-trainer-api:local
    working_dir: /app
    profiles: ["training"]
    networks: [proseqgo]
    environment: &trainer-env
      PYTHONUNBUFFERED: "1"
      PYTHONPATH: /app:/app/services/training-api
      CAFA_DEVICE: auto
      MLFLOW_TRACKING_URI: http://mlflow:5000
      MLFLOW_S3_ENDPOINT_URL: ${MLFLOW_S3_ENDPOINT_URL}
      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
      AWS_DEFAULT_REGION: us-east-1
      MLFLOW_S3_IGNORE_TLS: "true"
      MLFLOW_EXTERNAL_UI_BASE: http://127.0.0.1/mlflow
      REGISTERED_MODEL_NAME: ${REGISTERED_MODEL_NAME}
      PROMOTION_THRESHOLD: ${PROMOTION_THRESHOLD}
      JOBS_DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/proseqgo_jobs
      REDIS_URL: redis://redis:6379/0
      TRAINING_JOB_TIMEOUT_SEC: ${TRAINING_JOB_TIMEOUT_SEC:-86400}
      TRAINING_API_ARTIFACT_ROOT: /app/outputs/training_api
    depends_on:
      mlflow:
        condition: service_started
      postgres:
        condition: service_healthy
      redis:
        condition: service_healthy
    volumes:
      - ./data:/app/data
      - ./outputs:/app/outputs

  trainer-worker:
    build:
      context: .
      dockerfile: docker/docker_training/Dockerfile.training
    image: proseqgo-trainer-api:local
    working_dir: /app/services/training-api
    profiles: ["training"]
    networks: [proseqgo]
    stop_grace_period: 2h
    environment:
      <<: *trainer-env
      WORKER_METRICS_PORT: "8001"
    depends_on:
      trainer-api:
        condition: service_started
      redis:
        condition: service_healthy
      postgres:
        condition: service_healthy
    volumes:
      - ./data:/app/data
      - ./outputs:/app/outputs
    command: ["python", "rq_worker_entry.py"]
    restart: unless-stopped

  embedding-api:
    build:
      context: .
      dockerfile: docker/docker_embedding/Dockerfile.embedding-api
    image: proseqgo-embedding-api:local
    working_dir: /app/services/embedding-api
    networks: [proseqgo]
    environment: &embedding-env
      PYTHONUNBUFFERED: "1"
      PYTHONPATH: /app:/app/services/embedding-api
      CAFA_DEVICE: auto
      MLFLOW_TRACKING_URI: http://mlflow:5000
      MLFLOW_S3_ENDPOINT_URL: ${MLFLOW_S3_ENDPOINT_URL}
      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
      AWS_DEFAULT_REGION: us-east-1
      MLFLOW_S3_IGNORE_TLS: "true"
      GO_PREDICTION_API_URL: http://go-prediction-api:8000
      JOBS_DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/proseqgo_jobs
      REDIS_URL: redis://redis:6379/0
      EMBEDDING_JOB_TIMEOUT_SEC: ${EMBEDDING_JOB_TIMEOUT_SEC:-3600}
      EMBEDDING_ARTIFACT_ROOT: /app/outputs/service_artifacts
      MAX_SEQUENCES_PER_REQUEST: ${MAX_SEQUENCES_PER_REQUEST:-20}
      MAX_SEQUENCE_LENGTH_AA: ${MAX_SEQUENCE_LENGTH_AA:-1000}
      MAX_FASTA_UPLOAD_MB: ${MAX_FASTA_UPLOAD_MB:-2}
      SYNC_PREDICT_TIMEOUT_SEC: ${SYNC_PREDICT_TIMEOUT_SEC:-600}
      SYNC_PREDICT_POLL_INTERVAL_SEC: ${SYNC_PREDICT_POLL_INTERVAL_SEC:-1.0}
    depends_on:
      go-prediction-api:
        condition: service_started
      mlflow:
        condition: service_started
      postgres:
        condition: service_healthy
      redis:
        condition: service_healthy
    volumes:
      - ./outputs:/app/outputs
      - ./data/hf_cache:/app/data/hf_cache

  embedding-worker:
    build:
      context: .
      dockerfile: docker/docker_embedding/Dockerfile.embedding-api
    image: proseqgo-embedding-api:local
    working_dir: /app/services/embedding-api
    networks: [proseqgo]
    stop_grace_period: 3700s
    environment:
      <<: *embedding-env
      WORKER_METRICS_PORT: "8001"
      # Gives crash-recovery tests a reliable kill window after mark_running.
      EMBEDDING_JOB_START_DELAY_SEC: ${EMBEDDING_JOB_START_DELAY_SEC:-0}
    depends_on:
      embedding-api:
        condition: service_started
      redis:
        condition: service_healthy
      postgres:
        condition: service_healthy
    volumes:
      - ./outputs:/app/outputs
      - ./data/hf_cache:/app/data/hf_cache
    command: ["python", "rq_worker_entry.py"]
    restart: unless-stopped

  go-prediction-api:
    build:
      context: .
      dockerfile: docker/docker_go_term/Dockerfile.api
    image: proseqgo-go-prediction-api:local
    working_dir: /app/services/go-prediction-api
    networks: [proseqgo]
    environment:
      PYTHONUNBUFFERED: "1"
      PYTHONPATH: /app:/app/services/go-prediction-api
      CAFA_DEVICE: auto
      MLFLOW_TRACKING_URI: http://mlflow:5000
      MLFLOW_S3_ENDPOINT_URL: ${MLFLOW_S3_ENDPOINT_URL}
      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
      AWS_DEFAULT_REGION: us-east-1
      MLFLOW_S3_IGNORE_TLS: "true"
      REGISTERED_MODEL_NAME: ${REGISTERED_MODEL_NAME}
      MODEL_URI: models:/${REGISTERED_MODEL_NAME}@champion
      MODEL_CACHE_DIR: /app/model_cache
    depends_on:
      - mlflow
    volumes:
      - ./outputs:/app/outputs

  streamlit-ui:
    build:
      context: .
      dockerfile: docker/docker_streamlit/Dockerfile.streamlit
    image: proseqgo-streamlit-ui:local
    working_dir: /app/services/streamlit-ui
    networks: [proseqgo]
    environment:
      GATEWAY_BASE_URL: http://nginx
      # Predict-route credentials (public user). Must match make gateway-auth / .htpasswd-user.
      GATEWAY_USER: ${GATEWAY_USER:-user}
      GATEWAY_USER_PASSWORD: ${GATEWAY_USER_PASSWORD:-change-me-gateway-user}
      # Compose uses plain HTTP to nginx; disable TLS verification explicitly.
      GATEWAY_VERIFY_TLS: "false"
      MAX_SEQUENCES_PER_REQUEST: ${MAX_SEQUENCES_PER_REQUEST:-20}
      MAX_SEQUENCE_LENGTH_AA: ${MAX_SEQUENCE_LENGTH_AA:-1000}
      MAX_FASTA_UPLOAD_MB: ${MAX_FASTA_UPLOAD_MB:-2}
      SYNC_PREDICT_TIMEOUT_SEC: ${SYNC_PREDICT_TIMEOUT_SEC:-600}
    depends_on:
      - embedding-api
      - go-prediction-api
      - mlflow

  nginx:
    image: nginx:latest
    networks: [proseqgo]
    ports:
      - "80:80"
    volumes:
      - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro
      - ./nginx/.htpasswd-admin:/etc/nginx/.htpasswd-admin:ro
      - ./nginx/.htpasswd-user:/etc/nginx/.htpasswd-user:ro
    depends_on:
      - embedding-api
      - embedding-worker
      - go-prediction-api
      - mlflow
      - streamlit-ui

  redis-exporter:
    image: oliver006/redis_exporter:v1.66.0
    profiles: ["monitoring"]
    networks: [proseqgo]
    environment:
      REDIS_ADDR: redis://redis:6379
    depends_on:
      redis:
        condition: service_healthy
    restart: unless-stopped

  prometheus:
    image: prom/prometheus:latest
    container_name: prometheus
    profiles: ["monitoring"]
    networks: [proseqgo]
    command:
      - --config.file=/etc/prometheus/prometheus.yml
      - --storage.tsdb.retention.time=15d
      - --web.enable-lifecycle
    ports:
      - "127.0.0.1:9090:9090"
    restart: unless-stopped
    volumes:
      - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro
      - ./monitoring/alerts.yml:/etc/prometheus/alerts.yml:ro
      - prometheus_data:/prometheus
    depends_on:
      - embedding-api
      - embedding-worker
      - redis-exporter

  grafana:
    image: grafana/grafana:latest
    container_name: grafana
    profiles: ["monitoring"]
    networks: [proseqgo]
    environment:
      GF_SECURITY_ADMIN_USER: admin
      GF_SECURITY_ADMIN_PASSWORD: admin
    ports:
      - "127.0.0.1:3000:3000"
    restart: unless-stopped
    volumes:
      - grafana_data:/var/lib/grafana
      - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro
      - ./monitoring/grafana/dashboards:/etc/grafana/dashboards:ro