Skip to content

chore: Add Java 17 Hadoop base image and Spark 4.0.1 docker compose s… #18520

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yihua merged 1 commit into from
Apr 17, 2026
Merged

chore: Add Java 17 Hadoop base image and Spark 4.0.1 docker compose s… #18520

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion docker/build_docker_images.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,10 +83,21 @@ fi
# Docker image tags
LATEST_TAG="latest"
DOCKER_CONTEXT_DIR="hoodie/hadoop"

# Select Java base image based on Spark version (Spark 4.0+ requires Java 17)
SPARK_MAJOR=$(echo "$SPARK_VERSION" | cut -d. -f1)
Comment thread
yihua marked this conversation as resolved.
if [ "$SPARK_MAJOR" -ge 4 ] 2>/dev/null; then
BASE_IMAGE_DIR="base_java17"
echo "Using Java 17 base image for Spark ${SPARK_VERSION}"
else
BASE_IMAGE_DIR="base_java11"
echo "Using Java 11 base image for Spark ${SPARK_VERSION}"
fi

# List of images to build: "subdir|image_base_name"
# Each entry: <subdir>|<image_base_name>
DOCKER_IMAGES=(
"base_java11|apachehudi/hudi-hadoop_${HADOOP_VERSION}-base"
"${BASE_IMAGE_DIR}|apachehudi/hudi-hadoop_${HADOOP_VERSION}-base"
"datanode|apachehudi/hudi-hadoop_${HADOOP_VERSION}-datanode"
"historyserver|apachehudi/hudi-hadoop_${HADOOP_VERSION}-history"
"hive_base|apachehudi/hudi-hadoop_${HADOOP_VERSION}-hive_${HIVE_VERSION}"
Expand Down
264 changes: 264 additions & 0 deletions docker/compose/docker-compose_hadoop340_hive313_spark401_amd64.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,264 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

services:

namenode:
image: apachehudi/hudi-hadoop_3.4.0-namenode:latest
hostname: namenode
Comment thread
yihua marked this conversation as resolved.
container_name: namenode
environment:
- CLUSTER_NAME=hudi_hadoop340_hive313_spark401
ports:
- "50070:50070"
- "8020:8020"
- "9870:9870"
env_file:
- ./hadoop.env
healthcheck:
test: ["CMD", "curl", "-f", "http://namenode:9870"]
interval: 30s
timeout: 10s
retries: 3

Comment thread
yihua marked this conversation as resolved.
datanode1:
image: apachehudi/hudi-hadoop_3.4.0-datanode:latest
container_name: datanode1
hostname: datanode1
environment:
- CLUSTER_NAME=hudi_hadoop340_hive313_spark401
env_file:
- ./hadoop.env
ports:
- "50075:50075"
- "9864:9864"
- "50010:50010"
links:
- "namenode"
- "historyserver"
healthcheck:
test: ["CMD", "curl", "-f", "http://datanode1:9864"]
interval: 30s
timeout: 10s
retries: 3
depends_on:
- namenode

historyserver:
image: apachehudi/hudi-hadoop_3.4.0-history:latest
hostname: historyserver
container_name: historyserver
environment:
- CLUSTER_NAME=hudi_hadoop340_hive313_spark401
depends_on:
- "namenode"
links:
- "namenode"
ports:
- "8188:8188"
Comment thread
yihua marked this conversation as resolved.
healthcheck:
test: ["CMD", "curl", "-f", "http://historyserver:8188"]
interval: 30s
timeout: 10s
retries: 3
env_file:
- ./hadoop.env
volumes:
- historyserver:/hadoop/yarn/timeline

hive-metastore-postgresql:
image: bde2020/hive-metastore-postgresql:3.1.0
volumes:
- hive-metastore-postgresql:/var/lib/postgresql
hostname: hive-metastore-postgresql
container_name: hive-metastore-postgresql

hivemetastore:
image: apachehudi/hudi-hadoop_3.4.0-hive_3.1.3:latest
hostname: hivemetastore
container_name: hivemetastore
links:
- "hive-metastore-postgresql"
- "namenode"
env_file:
- ./hadoop.env
command: /opt/hive/bin/hive --service metastore
environment:
- "SERVICE_PRECONDITION=namenode:9870 hive-metastore-postgresql:5432"
ports:
- "9083:9083"
healthcheck:
test: ["CMD", "nc", "-z", "hivemetastore", "9083"]
interval: 30s
timeout: 10s
retries: 3
depends_on:
- "hive-metastore-postgresql"
- "namenode"

hiveserver:
image: apachehudi/hudi-hadoop_3.4.0-hive_3.1.3:latest
hostname: hiveserver
container_name: hiveserver
env_file:
- ./hadoop.env
environment:
- SERVICE_PRECONDITION=hivemetastore:9083
ports:
- "10000:10000"
- "10002:10002"
depends_on:
- "hivemetastore"
links:
- "hivemetastore"
- "hive-metastore-postgresql"
- "namenode"
volumes:
- ${HUDI_WS}:/var/hoodie/ws

zookeeper:
image: 'bitnamilegacy/zookeeper:3.6.4'
hostname: zookeeper
container_name: zookeeper
ports:
- "2181:2181"
environment:
- ALLOW_ANONYMOUS_LOGIN=yes

kafka:
image: 'bitnamilegacy/kafka:3.4.1'
Copy link
Copy Markdown
Contributor

@yihua yihua Apr 17, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Non-blocker: should we consider using 'apache/kafka:3.7.2'?

hostname: kafkabroker
container_name: kafkabroker
ports:
- "9092:9092"
environment:
- KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
- ALLOW_PLAINTEXT_LISTENER=yes
Comment on lines +147 to +148
Copy link
Copy Markdown
Contributor

@yihua yihua Apr 17, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Non-blocker: do we need same configs as Spark3.5 docker image (docker-compose_hadoop334_hive313_spark353_arm64):

      - KAFKA_NODE_ID=1
      - KAFKA_PROCESS_ROLES=broker,controller
      - KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:29092,CONTROLLER://0.0.0.0:9093,PLAINTEXT_HOST://0.0.0.0:9092
      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafkabroker:29092,PLAINTEXT_HOST://localhost:9092
      - KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER
      - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT
      - KAFKA_CONTROLLER_QUORUM_VOTERS=1@kafkabroker:9093
      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
      - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1
      - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1
      - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
      - KAFKA_NUM_PARTITIONS=3


sparkmaster:
image: apachehudi/hudi-hadoop_3.4.0-hive_3.1.3-sparkmaster_4.0.1:latest
hostname: sparkmaster
container_name: sparkmaster
env_file:
- ./hadoop.env
ports:
- "8080:8080"
- "7077:7077"
- "8888:8888"
volumes:
- ${HUDI_WS}:/var/hoodie/ws
- ./notebooks:/opt/workspace/notebooks
environment:
- INIT_DAEMON_STEP=setup_spark
links:
- "hivemetastore"
- "hiveserver"
- "hive-metastore-postgresql"
- "namenode"

spark-worker-1:
image: apachehudi/hudi-hadoop_3.4.0-hive_3.1.3-sparkworker_4.0.1:latest
hostname: spark-worker-1
container_name: spark-worker-1
env_file:
- ./hadoop.env
depends_on:
- sparkmaster
ports:
- "8081:8081"
environment:
- SPARK_MASTER=spark://sparkmaster:7077
links:
- "hivemetastore"
- "hiveserver"
- "hive-metastore-postgresql"
- "namenode"

adhoc-1:
image: apachehudi/hudi-hadoop_3.4.0-hive_3.1.3-sparkadhoc_4.0.1:latest
hostname: adhoc-1
container_name: adhoc-1
env_file:
- ./hadoop.env
depends_on:
- sparkmaster
ports:
- '4040:4040'
environment:
- SPARK_MASTER=spark://sparkmaster:7077
links:
- "hivemetastore"
- "hiveserver"
- "hive-metastore-postgresql"
- "namenode"
volumes:
- ${HUDI_WS}:/var/hoodie/ws

adhoc-2:
image: apachehudi/hudi-hadoop_3.4.0-hive_3.1.3-sparkadhoc_4.0.1:latest
hostname: adhoc-2
container_name: adhoc-2
env_file:
- ./hadoop.env
depends_on:
- sparkmaster
environment:
- SPARK_MASTER=spark://sparkmaster:7077
links:
- "hivemetastore"
- "hiveserver"
- "hive-metastore-postgresql"
- "namenode"
volumes:
- ${HUDI_WS}:/var/hoodie/ws

minio:
image: 'minio/minio:latest'
hostname: minio
container_name: minio
ports:
- 9090:9090 # server address
- 9091:9091 # console address
volumes:
- minio-data:/data
environment:
- MINIO_ACCESS_KEY=minio
- MINIO_SECRET_KEY=minio123
- MINIO_DOMAIN=minio
command: server --address ":9090" --console-address ":9091" /data

mc:
image: minio/mc
container_name: mc
entrypoint: >
/bin/sh -c "
until (/usr/bin/mc alias set minio http://minio:9090 minio minio123 --api S3v4) do echo '...waiting...' && sleep 1; done;
/usr/bin/mc rm -r --force minio/warehouse;
/usr/bin/mc mb minio/warehouse;
/usr/bin/mc policy set public minio/warehouse;
tail -f /dev/null
"
depends_on:
- minio

volumes:
namenode:
historyserver:
hive-metastore-postgresql:
minio-data:

networks:
default:
name: hudi
Loading
Loading