From 8f62d8de19fd0c76b6880c7e3fe221cb3fd57982 Mon Sep 17 00:00:00 2001 From: Joe Date: Wed, 22 Apr 2026 09:40:23 +0200 Subject: [PATCH] fix(notebooks): pyspark increase resources --- notebooks/nyc_trip/raw_pyspark.ipynb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/notebooks/nyc_trip/raw_pyspark.ipynb b/notebooks/nyc_trip/raw_pyspark.ipynb index 5fc3fd1..05db5c6 100644 --- a/notebooks/nyc_trip/raw_pyspark.ipynb +++ b/notebooks/nyc_trip/raw_pyspark.ipynb @@ -109,7 +109,9 @@ "spark = (\n", " SparkSession.builder\n", " .appName(\"NYC Tripdata — PySpark\")\n", - " .config(\"spark.executor.memory\", \"2000M\")\n", + " .config(\"spark.executor.memory\", \"4G\")\n", + " .config(\"spark.driver.memory\", \"2G\")\n", + " .config(\"spark.kubernetes.memoryOverheadFactor\", \"0.3\")\n", " .config(\"spark.executor.cores\", \"2\")\n", " .config(\"spark.executor.instances\", \"1\")\n", " .config(\"spark.hadoop.fs.s3a.endpoint\", \"https://seaweedfs-seaweedfs-default.okdp.sandbox:443\")\n",