docker-compose.yml

services:

  broker:
    image: apache/kafka:latest
    hostname: cf-broker
    container_name: cf-broker
    ports:
      - "9092:9092"
    environment:
      KAFKA_DELETE_TOPIC_ENABLE: "true"
      KAFKA_NODE_ID: 1
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT'
      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT_HOST://localhost:9092,PLAINTEXT://cf-broker:19092'
      KAFKA_PROCESS_ROLES: 'broker,controller'
      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@cf-broker:29093'
      KAFKA_LISTENERS: 'CONTROLLER://:29093,PLAINTEXT_HOST://:9092,PLAINTEXT://:19092'
      KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT'
      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
      CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_MIN_ISR: 1
      KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
    volumes:
      - ./kafka/:/home/appuser/

  fake-stream: 
    build: 
      context: ./fake-stream
      dockerfile: Dockerfile
    container_name: cf-fake-stream
    hostname: cf-fake-stream
    ports: 
      - '7777:7777'
    volumes:
      - ./fake-stream:/opt/fake-stream
    environment:
      - HOME=/opt/fake-stream
    command: sleep infinity
    
  spark-clean:
    build:
      context: ./spark-clean
      dockerfile: Dockerfile
    image: apache/spark:3.4.4
    container_name: cf-spark-clean
    hostname: cf-spark-clean
    environment:
      - SPARK_MODE=master
      - HOME=/opt/spark/work-dir
    ports:
      - "4040:4040"  # UI Spark Web
    volumes:
      - ./spark-clean/work-dir:/opt/spark/work-dir
      - ./spark-clean/conf:/opt/spark/conf
      - ./spark-clean/ivy-cache:/home/spark/.ivy2
    command: sleep infinity

  spark-flatten:
    image: apache/spark:3.4.4
    container_name: cf-spark-flatten
    hostname: cf-spark-flatten
    environment:
      - SPARK_MODE=master
      - HOME=/opt/spark/work-dir
    ports:
      - "4041:4040"  # UI Spark Web
    volumes:
      - ./spark-flatten/ivy-cache:/home/spark/.ivy2 
      - ./spark-flatten/conf:/opt/spark/conf/
      - ./spark-flatten/work-dir/:/opt/spark/work-dir
    command: sleep infinity

  spark-clickhouse:
    image: apache/spark:3.4.4
    container_name: cf-spark-clickhouse
    hostname: cf-spark-clickhouse
    environment:
      - SPARK_MODE=master
      - HOME=/opt/spark/work-dir
    ports:
      - "4045:4040"  # UI Spark Web
    volumes:
      - ./spark-flatten/ivy-cache:/home/spark/.ivy2 
      - ./spark-clickhouse:/opt/spark/work-dir
    command: sleep infinity

  grafana:
    image: grafana/grafana:latest
    container_name: cf-grafana
    hostname: cf-grafana
    ports:
      - "3000:3000"
    environment:
      GF_SECURITY_ADMIN_PASSWORD: admin
      GF_AUTH_ANONYMOUS_ENABLED: 'true'
      GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer
      GF_AUTH_ANONYMOUS_ORG_ID: 1
    volumes:
      - ./grafana/conf:/usr/share/grafana/conf
      - grafana-data:/var/lib/grafana

  clickhouse:
    image: clickhouse/clickhouse-server:latest
    container_name: cf-clickhouse
    hostname: cf-clickhouse
    environment:
      HOME: /home/clickhouse
      CLICKHOUSE_USER: root
      CLICKHOUSE_PASSWORD: root
      CLICKHOUSE_DB: default
    ports:
      - "8123:8123"  # HTTP
      - "9000:9000"  # Native
    volumes:
      - ./clickhouse:/home/clickhouse/
      - clickhouse-data:/var/lib/CLICKHOUSE_DB

  spark-cleaned:
    image: apache/spark:3.4.4
    container_name: cf-spark-cleaned
    hostname: cf-spark-cleaned
    environment:
      - SPARK_MODE=master
      - HOME=/opt/spark/work-dir
    ports:
      - "4042:4040"  # UI Spark Web
    volumes:
      - ./spark-cleaned:/opt/spark/work-dir
      - ./spark-cleaned/ivy-cache:/home/spark/.ivy2
    command: sleep infinity

  
  
  postgres:
    image: postgres:17
    container_name: cf-postgres
    hostname: cf-postgres
    environment:
      POSTGRES_USER: airflow
      POSTGRES_PASSWORD: airflow
      POSTGRES_DB: airflow
    volumes:
      - pg_data:/var/lib/postgresql/data

  airflow:
    image: apache/airflow:2.11.0
    container_name: cf-airflow
    hostname: cf-airflow
    depends_on:
      - postgres
    environment:  
      _AIRFLOW_WWW_USER_USERNAME: cf
      _AIRFLOW_WWW_USER_PASSWORD: cf
      AIRFLOW__CORE__AUTH_MANAGER: airflow.providers.fab.auth_manager.fab_auth_manager.FabAuthManager
      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@cf-postgres:5432/airflow
      AIRFLOW__CORE__EXECUTOR: LocalExecutor
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - ./airflow/dags:/opt/airflow/dags
      - ./airflow/plugins:/opt/airflow/plugins
      - ./airflow/config:/opt/airflow/config
      - ./airflow/requirements.txt:/requirements.txt
      # - ./airflow/simple_auth_manager_passwords.json.generated:/opt/airflow/simple_auth_manager_passwords.json.generated
    command: >
      bash -c "
        pip install -r /requirements.txt &&
        airflow db migrate &&
        airflow users create --email email --firstname firstname --lastname lastname --password cf --role Admin --username cf &&
        airflow standalone
      "

    # command: >
    #   bash -c "
    #     pip install -r /requirements.txt &&
    #     airflow db migrate &&
    #     airflow standalone
    #   "
    ports:
      - "8080:8080"

  spark-transform:
    image: apache/spark:3.4.4
    container_name: cf-spark-transform
    hostname: cf-spark-transform
    environment:
      - SPARK_MODE=master
      - HOME=/opt/spark/work-dir
    ports:
      - "4043:4040"  # UI Spark Web
    volumes:
      - ./spark-transform:/opt/spark/work-dir/
      - ./spark-flatten/ivy-cache:/home/spark/.ivy2
    command: sleep infinity


  minio:
    image: minio/minio
    container_name: cf-minio
    hostname: cf-minio
    ports:
      - "9049:9000"   # S3 API
      - "9050:9001"   # Web UI
    environment:
      - MINIO_ROOT_USER=minio
      - MINIO_ROOT_PASSWORD=minio123
    command: server /data --console-address ":9001"
    volumes:
      - minio_data:/data

  init: 
    image: python:3.11
    container_name: cf-init
    hostname: cf-init
    volumes:
      - ./init:/opt/init
    working_dir: /opt/init
    command: >
      bash -c "
        pip install -r requirements.txt && configs:
        sleep infinity
      "


volumes:
  clickhouse-data:
  grafana-data:
  airflow-db-volume:
  pg_data:
  minio_data:
  superset_home: