mlflow-postgres-minio/docker-compose.yml at main · pandego/mlflow-postgres-minio · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
---
networks:
  frontend:
    driver: bridge
  backend:
    driver: bridge


# Uncomment if you want docker to fully manage the volumes
#volumes:
#  db_data:
#  minio_data:


services:
  db:
    restart: unless-stopped
    image: postgres:${PG_VERSION}
    container_name: mlflow_db
    ports:
      - "${PG_PORT_EXPOSED}:${PG_PORT_SERVICE}"
    networks:
      - backend
    environment:
      - POSTGRES_USER=${PG_USER}
      - POSTGRES_PASSWORD=${PG_PASSWORD}
      - POSTGRES_DATABASE=${PG_DATABASE}
    volumes:
      - ${DATA_DIR}/db_data:/var/lib/postgresql/data/
      # - db_data:/var/lib/postgresql/data/  # Uncomment if you want docker to fully manage the volumes
    command: ["postgres", "-p", "${PG_PORT_SERVICE}"]
    healthcheck:
      test: ["CMD", "pg_isready", "-p", "${PG_PORT_SERVICE}", "-U", "${PG_USER}"]
      interval: 5s
      timeout: 5s
      retries: 3

  s3:
    restart: unless-stopped
    image: minio/minio:${MINIO_VERSION}
    container_name: mlflow_minio
    volumes:
      - ${DATA_DIR}/minio_data:/data
      # - minio_data:/data  # Uncomment if you want docker to fully manage the volumes
    ports:
      - "${MINIO_PORT_API}:9000"  # API
      - "${MINIO_PORT_UI}:9001"   # WebUI
    networks: ['frontend', 'backend']
    environment:
      - MINIO_ROOT_USER=${MINIO_ROOT_USER}
      - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD}
      # - MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY}          # deprecated
      # - MINIO_SECRET_KEY=${MINIO_SECRET_ACCESS_KEY}   # deprecated
    command: server /data --console-address ":9001" --address ":9000"
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
      interval: 30s
      timeout: 20s
      retries: 3

  create_buckets:
    image: minio/mc:${MINIO_VERSION}
    container_name: mlflow_create_buckets
    depends_on:
      s3:
        condition: service_healthy
    networks:
      - backend
    entrypoint: >
      /bin/sh -c '
      sleep 5;
      /usr/bin/mc config host add s3 http://s3:${MINIO_PORT_API} ${MINIO_ROOT_USER} ${MINIO_ROOT_PASSWORD} --api S3v4;
      [[ ! -z "`/usr/bin/mc ls s3 | grep challenge`" ]] || /usr/bin/mc mb s3/${MLFLOW_BUCKET_NAME};
      /usr/bin/mc policy download s3/${MLFLOW_BUCKET_NAME};
      [[ ! -z "`/usr/bin/mc ls s3 | grep challenge`" ]] || /usr/bin/mc mb s3/${DATA_REPO_BUCKET_NAME};
      /usr/bin/mc policy download s3/${DATA_REPO_BUCKET_NAME};
      exit 0;
      '

  tracking_server:
    restart: unless-stopped
    build: ./mlflow
    image: mlflow_server
    container_name: mlflow_server
    ports:
      - "${MLFLOW_PORT_EXPOSED}:${MLFLOW_PORT_SERVICE}"
    networks: ['frontend', 'backend']
    environment:
      - AWS_ACCESS_KEY_ID=${MINIO_ROOT_USER}
      - AWS_SECRET_ACCESS_KEY=${MINIO_ROOT_PASSWORD}
      - MLFLOW_S3_ENDPOINT_URL=http://s3:${MINIO_PORT_API}
    command: >
      mlflow server
      --backend-store-uri postgresql://${PG_USER}:${PG_PASSWORD}@db:${PG_PORT_SERVICE}/${PG_DATABASE}
      --host 0.0.0.0
      --port ${MLFLOW_PORT_SERVICE}
      --default-artifact-root s3://mlflow/
    depends_on:
      db:
        condition: service_healthy
      s3:
        condition: service_healthy
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:${MLFLOW_PORT_SERVICE}/"]
      interval: 30s
      timeout: 10s
      retries: 3