docker-compose.yml

x-airflow-common:
  &airflow-common
  build:
    context: ./airflow
    dockerfile: airflow.Dockerfile
  user: "0:0"  # run as root to access docker.sock
  environment:
    &airflow-common-env
    AIRFLOW__CORE__EXECUTOR: LocalExecutor
    AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow
    AIRFLOW__WEBSERVER__SECRET_KEY: "super-secret-key-please-change-me"  # use the same in all services!
    AIRFLOW__CORE__FERNET_KEY: "X5uLmpuVDfYEm1LXYGxR8UwHAPEVIXKo5A98O4yU8lY="  # optional but good to define
    AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'false'
    AIRFLOW__CORE__LOAD_EXAMPLES: 'false'
    AIRFLOW_CONN_SSH_SPARK_MASTER: ssh://root:root@spark-master:22
    AIRFLOW_CONN_SSH_DBT: ssh://root:root@dbt:22
  volumes:
    - ./airflow/dags:/opt/airflow/dags
    - ./airflow/logs:/opt/airflow/logs
    - ./airflow/plugins:/opt/airflow/plugins
    # - /var/run/docker.sock:/var/run/docker.sock  # allows running host docker
  depends_on:
    postgres:
      condition: service_healthy

services:
  hadoop-master:
    build: ./hadoop
    hostname: hadoop-master
    volumes:
      - ./hadoop/start-namenode.sh:/start-namenode.sh
      - hadoop_data_master:/opt/hadoop/data/namenode
    ports:
      - "9871:9870"  # HDFS Web UI
      - "8089:8088"
    command: ["/bin/bash", "/start-namenode.sh"]
  
  hadoop-slave1:
    build: ./hadoop
    hostname: hadoop-slave1
    volumes:
      - ./hadoop/start-datanode.sh:/start-datanode.sh
      - hadoop_data_slave1:/opt/hadoop/data/datanode
    command: ["/bin/bash", "/start-datanode.sh"]
    

  hadoop-slave2:
    build: ./hadoop
    hostname: hadoop-slave2
    volumes:
      - ./hadoop/start-datanode.sh:/start-datanode.sh
      - hadoop_data_slave2:/opt/hadoop/data/datanode
    command: ["/bin/bash", "/start-datanode.sh"]
    
  spark-master:
    build: ./spark
    hostname: spark-master
    volumes:
      - ./spark-app:/opt/spark-app
      - ./spark/start-master.sh:/start-master.sh
    command: ["/bin/bash", "/start-master.sh"]
    env_file:
      - .env
    ports:
      - "8080:8080"
      - "7077:7077"
  spark-worker-1:
    build: ./spark
    hostname: spark-worker-1
    volumes:
      - ./spark/start-worker.sh:/start-worker.sh
    command: ["/bin/bash", "/start-worker.sh"]
    depends_on:
      - spark-master

  spark-worker-2:
    build: ./spark
    hostname: spark-worker-2
    volumes:
      - ./spark/start-worker.sh:/start-worker.sh
    command: ["/bin/bash", "/start-worker.sh"]
    depends_on:
      - spark-master
  spark-worker-3:
    build: ./spark
    hostname: spark-worker-3
    volumes:
      - ./spark/start-worker.sh:/start-worker.sh
    command: ["/bin/bash", "/start-worker.sh"]
    depends_on:
      - spark-master
  
  mysql:
    image: mysql:8
    volumes:
      - mysql-data:/var/lib/mysql
      # - ./mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
    environment:
      MYSQL_ROOT_PASSWORD: root
      MYSQL_DATABASE: metastore
    ports:
      - "3307:3306"
    healthcheck:
      test: ["CMD", "mysqladmin", "ping", "-h", "localhost"]
      interval: 5s
      timeout: 5s
      retries: 5
    

  hive-metastore:
    build: ./hive-metastore
    hostname: hive-metastore
    ports:
      - "9083:9083"
    depends_on:
      mysql:
        condition: service_healthy
    volumes:
      - ./hive-metastore/conf/hive-site.xml:/opt/hive/conf/hive-site.xml
      - ./hive-metastore/init.sh:/init.sh
    command: ["/bin/bash", "/init.sh"]
    healthcheck:
      test: ["CMD", "nc", "-z", "localhost", "9083"]
      interval: 5s
      timeout: 5s
      retries: 5

  dbt:
    build:
      context: ./dbt
      dockerfile: dbt.Dockerfile
    hostname: dbt
    ports: 
      - "7748:7748"
    env_file:
      - .env
    volumes:
      - ./dbt:/dbt            # Nơi chứa project dbt của bạn trên host
    depends_on:
      - spark-master
      - hive-metastore
      - mysql
    command: ["/bin/bash", "/dbt/setup_dbt.sh"]
  
  postgres:
    image: postgres:13
    environment:
      POSTGRES_USER: airflow
      POSTGRES_PASSWORD: airflow
      POSTGRES_DB: airflow
    volumes:
      - postgres-db-volume:/var/lib/postgresql/data
    healthcheck:
      test: ["CMD", "pg_isready", "-U", "airflow"]
      interval: 5s
      retries: 5

  airflow-init:
    <<: *airflow-common
    command: >
      bash -c "
        airflow db upgrade &&
        airflow users create --username admin --password admin --firstname Admin --lastname User --role Admin --email admin@example.com
      "

  airflow-webserver:
    <<: *airflow-common
    ports:
      - 8081:8080
    command: webserver
    depends_on:
      airflow-init:
        condition: service_completed_successfully

  airflow-scheduler:
    <<: *airflow-common
    command: scheduler
    depends_on:
      airflow-init:
        condition: service_completed_successfully

volumes:
  hadoop_data_master:
  hadoop_data_slave1:
  hadoop_data_slave2:
  mysql-data:
  postgres-db-volume: