opensnowcat-devkit/docker-compose.yml at main · opensnowcat/opensnowcat-devkit · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
networks:
  appnet:
    name: opensnowcat

services:

  # Just for quick compatibility, not having to change the
  # collector and enrich configs, this is still Kafka
  warp:
    image: apache/kafka:latest
    container_name: warp
    restart: on-failure
    networks:
      appnet:
        aliases: [warp]
    # no host port needed for container-to-container, but keep if you want host access:
    ports:
      - "29092:29092"  # optional; not used by other containers
    environment:
      # --- KRaft single node ---
      KAFKA_NODE_ID: 1
      KAFKA_PROCESS_ROLES: broker,controller
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@warp:9093

      # --- One data listener on 9092 + controller on 9093 ---
      KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://warp:9092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT

      # --- Single-broker internal topics must be RF=1 ---
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_MIN_INSYNC_REPLICAS: 1

      # --- Your tuning (optional) ---
      KAFKA_LOG_RETENTION_HOURS: 168
      KAFKA_MESSAGE_MAX_BYTES: 200000000
      KAFKA_REPLICA_FETCH_MAX_BYTES: 200000000
      KAFKA_MAX_REQUEST_SIZE: 200000000

  opensnowcat_collector:
    container_name: opensnowcat_collector
    networks: [ appnet ]
    image: opensnowcat/opensnowcat-collector-kafka:latest
    volumes:
      - ${PWD}/opensnowcat:/opensnowcat
    command: '--config /opensnowcat/config.collector.hocon'
    restart: on-failure
    depends_on:
      - warp
    ports:
      - "8080:8080"
    environment:
      LOG_LEVEL: "INFO"

  opensnowcat_enrich:
    container_name: opensnowcat_enrich
    networks: [ appnet ]
    image: opensnowcat/opensnowcat-enrich-kafka:latest
    volumes:
      - ${PWD}/opensnowcat:/opensnowcat
    command: '--enrichments /opensnowcat/enrichments --iglu-config /opensnowcat/resolver.json --config /opensnowcat/config.enrich.hocon'
    restart: on-failure
    depends_on:
      - warp
    environment:
      LOG_LEVEL: "INFO"

  kafka-ui:
    container_name: kafka-ui
    networks: [ appnet ]
    image: provectuslabs/kafka-ui:latest
    ports:
      - "8081:8080"
    depends_on:
      - warp
    environment:
      - KAFKA_CLUSTERS_0_NAME=Local Kafka Cluster
      - KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=warp:9092
      - KAFKA_CLUSTERS_0_ZOOKEEPER=zookeeper:2181
      - KAFKA_CLUSTERS_0_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT
      - DYNAMIC_CONFIG_ENABLED='true'

  kafka_proxy:
    container_name: kafka_proxy
    networks: [ appnet ]
    image: alpine/socat
    command: TCP-LISTEN:9092,fork,reuseaddr TCP:warp:9092
    ports:
      - "9092:9092"
    depends_on:
      - warp

  bento:
    container_name: bento
    networks: [ appnet ]
    image: ghcr.io/warpstreamlabs/bento:latest
    volumes:
      - ./bento:/configs
    command: '-c /configs/tsv-json.yml'
    restart: on-failure
    depends_on:
      - warp
      - opensnowcat_enrich
    environment:
      LOG_LEVEL: "DEBUG"