docker-compose.yml

services:
  archive_generator:
    build: ./mno_data_source_simulator
    command: python generate_archive.py --days ${ARCHIVE_DAYS:-1} --interval-seconds ${ARCHIVE_INTERVAL_SECONDS:-10}
    environment:
      - ARCHIVE_OUTPUT_DIR=/archive_output
      - NETCDF_FILE=/app/example_data/openMRG_cmls_20150827_3months.nc
      - NETCDF_FILE_URL=https://bwsyncandshare.kit.edu/s/jSAFftGXcJjQbSJ/download
    volumes:
      - ./parser/example_data:/app/example_data  # writable so the file can be downloaded
      - archive_data:/archive_output

  mno_simulator:
    build: ./mno_data_source_simulator
    restart: unless-stopped
    depends_on:
      - sftp_receiver
    volumes:
      - ./parser/example_data:/app/example_data  # writable so the file can be downloaded
      - mno_data_to_upload:/app/data_to_upload
      - mno_data_uploaded:/app/data_uploaded
      - ./ssh_keys:/app/ssh_keys:ro
    environment:
      - NETCDF_FILE=/app/example_data/openMRG_cmls_20150827_3months.nc
      - NETCDF_FILE_URL=https://bwsyncandshare.kit.edu/s/jSAFftGXcJjQbSJ/download
      - SFTP_HOST=sftp_receiver
      - SFTP_PORT=22
      - SFTP_USERNAME=cml_user
      - SFTP_REMOTE_PATH=/uploads
      - SFTP_USE_SSH_KEY=true
      - SFTP_PRIVATE_KEY_PATH=/app/ssh_keys/id_rsa
      - SFTP_KNOWN_HOSTS_PATH=/app/ssh_keys/known_hosts

  parser:
    build: ./parser
    ports:
      - "5004:5004"
    depends_on:
      database:
        condition: service_healthy
    environment:
      - DATABASE_URL=postgresql://myuser:mypassword@database:5432/mydatabase
      - PARSER_INCOMING_DIR=/app/data/incoming
      - PARSER_ARCHIVED_DIR=/app/data/archived
      - PARSER_QUARANTINE_DIR=/app/data/quarantine
      - PARSER_ENABLED=true
      - PROCESS_EXISTING_ON_STARTUP=true
      - LOG_LEVEL=INFO
    volumes:
      - sftp_uploads:/app/data/incoming
      - parser_archived:/app/data/archived
      - parser_quarantine:/app/data/quarantine

  archive_loader:
    image: timescale/timescaledb:latest-pg13
    depends_on:
      archive_generator:
        condition: service_completed_successfully
      database:
        condition: service_healthy
    environment:
      - PGHOST=database
      - PGUSER=myuser
      - PGPASSWORD=mypassword
      - PGDATABASE=mydatabase
      - ARCHIVE_DATA_DIR=/archive_data
    volumes:
      - archive_data:/archive_data:ro
      - ./database/init_archive_data.sh:/init_archive_data.sh:ro
    command: ["/bin/bash", "/init_archive_data.sh"]
    restart: "no"

  database:
    build: ./database
    ports:
      - "5432:5432"
    # Tune PostgreSQL memory for an 8 GB VM.
    # shared_buffers: keep recently-used chunks in RAM (default is only 128 MB).
    # effective_cache_size: hints to the planner how much OS page cache is available.
    # work_mem: memory per sort/hash operation; speeds up ORDER BY on large result sets.
    # random_page_cost: tell the planner data is effectively cached, prefer index scans.
    command: >
      postgres
        -c shared_buffers=2GB
        -c effective_cache_size=4GB
        -c work_mem=64MB
        -c maintenance_work_mem=256MB
        -c random_page_cost=1.1
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U myuser -d mydatabase"]
      interval: 5s
      timeout: 5s
      retries: 10
      start_period: 10s

  processor:
    build: ./processor
    ports:
      - "5002:5002"
    depends_on:
      - database
    environment:
      - DATABASE_URL=postgresql://myuser:mypassword@database:5432/mydatabase

  webserver:
    build: ./webserver
    ports:
      - "5000:5000"
    depends_on:
      - database
      - sftp_receiver
    environment:
      - DATABASE_URL=postgresql://myuser:mypassword@database:5432/mydatabase
      - STORAGE_BACKEND=local  # Options: local, s3, minio
      - STORAGE_BASE_PATH=/app/data
    volumes:
      - sftp_uploads:/app/data/incoming:ro  # Read-only access to SFTP uploads
      - webserver_data_staged:/app/data/staged
      - webserver_data_archived:/app/data/archived

  sftp_receiver:
    image: atmoz/sftp:latest
    ports:
      - "2222:22"
    volumes:
      - sftp_uploads:/home/cml_user/uploads
      - ./ssh_keys/sftp_host_ed25519_key:/etc/ssh/ssh_host_ed25519_key:ro
      - ./ssh_keys/sftp_host_rsa_key:/etc/ssh/ssh_host_rsa_key:ro
      - ./ssh_keys/authorized_keys:/home/cml_user/.ssh/keys/authorized_keys:ro
      - ./sftp_receiver/entrypoint.sh:/custom-entrypoint.sh:ro
    entrypoint: ["/custom-entrypoint.sh"]
    command: cml_user::1001:1001:uploads

  # Optional: MinIO for S3-compatible object storage (uncomment to use)
  # minio:
  #   image: minio/minio:latest
  #   ports:
  #     - "9000:9000"
  #     - "9001:9001"
  #   environment:
  #     MINIO_ROOT_USER: minioadmin
  #     MINIO_ROOT_PASSWORD: minioadmin
  #   volumes:
  #     - minio_data:/data
  #   command: server /data --console-address ":9001"
  #   healthcheck:
  #     test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
  #     interval: 30s
  #     timeout: 20s
  #     retries: 3

  grafana:
    image: grafana/grafana:latest
    ports:
      - "3000:3000"
    depends_on:
      - database
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin
      - GF_INSTALL_PLUGINS=grafana-piechart-panel
      - GF_USERS_ALLOW_SIGN_UP=false
      - GF_AUTH_ANONYMOUS_ENABLED=true
      - GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer
      - GF_SECURITY_ALLOW_EMBEDDING=true
      - GF_SERVER_ROOT_URL=http://localhost:5000/grafana/
      - GF_SERVER_SERVE_FROM_SUB_PATH=true
      - GF_AUTH_DISABLE_LOGIN_FORM=true
      - GF_SECURITY_COOKIE_SAMESITE=none
      - GF_SECURITY_COOKIE_SECURE=false
    volumes:
      - grafana_data:/var/lib/grafana
      - ./grafana/provisioning:/etc/grafana/provisioning

  # Integration test runner (run with: docker compose run --rm integration_tests)
  integration_tests:
    build:
      context: .
      dockerfile: tests/Dockerfile
    depends_on:
      - sftp_receiver
      - webserver
      - mno_simulator
    environment:
      - SFTP_HOST=sftp_receiver
      - SFTP_PORT=22
    volumes:
      - ./tests:/workspace/tests:ro
      - ./ssh_keys:/workspace/ssh_keys:ro
      - ./pyproject.toml:/workspace/pyproject.toml:ro
    network_mode: "service:sftp_receiver"
    profiles:
      - testing

volumes:
  archive_data:
  sftp_uploads:
  webserver_data_staged:
  webserver_data_archived:
  grafana_data:
  mno_data_to_upload:
  mno_data_uploaded:
  parser_archived:
  parser_quarantine:
  # minio_data:  # Uncomment if using MinIO