experimental_platform/spark-defaults.conf at main · vishal-labade/experimental_platform · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions
spark.master=spark://10.0.0.80:7077
spark.sql.defaultCatalog=exp


# ---------------------------
# Iceberg Catalog (Hadoop)
# ---------------------------
spark.sql.catalog.exp=org.apache.iceberg.spark.SparkCatalog
spark.sql.catalog.exp.type=hadoop
spark.sql.catalog.exp.warehouse=s3a://iceberg-warehouse/warehouse/


# ---------------------------
# S3A / MinIO config
# ---------------------------
spark.hadoop.fs.s3a.endpoint=http://10.0.0.80:9100
spark.hadoop.fs.s3a.access.key=minioadmin
spark.hadoop.fs.s3a.secret.key=minioadmin
spark.hadoop.fs.s3a.path.style.access=true
spark.hadoop.fs.s3a.connection.ssl.enabled=false
spark.hadoop.fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem

# ---------------------------
# Required JARs
# ---------------------------
# spark.jars=/opt/iceberg-jars/iceberg-spark-runtime-3.4_2.12-1.6.0.jar,/opt/iceberg-jars/iceberg-aws-bundle-1.6.0.jar

# ---------------------------
# Performance (optional but safe)
# ---------------------------
spark.sql.shuffle.partitions=200
spark.driver.memory=4g
spark.executor.memory=4g

spark.jars=/opt/runner-jars/iceberg-spark-runtime-3.4_2.12-1.6.0.jar,\
/opt/runner-jars/iceberg-aws-bundle-1.6.0.jar,\
/opt/runner-jars/hadoop-aws-3.3.4.jar,\
/opt/runner-jars/aws-java-sdk-bundle-1.12.262.jar


# # Temp:
# spark.sql.catalog.exp2=org.apache.iceberg.spark.SparkCatalog
# spark.sql.catalog.exp2.type=hadoop
# spark.sql.catalog.exp2.warehouse=s3a://iceberg-warehouse/warehouse_exp2/