Finalize Iceberg root and shim class placement [reduced-it]

dist/build/package-parallel-worlds.py

@@ -204,6 +204,10 @@ def root_safe_module_class_members(classifier):
 root_safe_modules = read_patterns(os.sep.join([dist_dir, 'root-safe-module-classes.txt']))
 from_single_shim_or_each = from_single_shim + from_each
 iceberg_audit_runtimes = {}
+# Classifiers are processed newest-first. Let older classifiers contribute
+# conditional root-safe classes that are absent from newer classifiers, without
+# overwriting a newer implementation of the same class path.
+promoted_root_safe_members = set()
 
 for bv in buildver_list:
     classifier = 'spark' + bv
@@ -225,7 +229,7 @@ def root_safe_module_class_members(classifier):
                 # IMPORTANT unconditional extract from the highest Spark version to the top
                 if bv == buildver_list[0] and art == 'sql-plugin-api':
                     zip_handle.extractall(path=top_dist_jar_dir)
-                if bv == buildver_list[0] and art == 'aggregator':
+                if art == 'aggregator':
                     namelist = zip_handle.namelist()
                     namelist_set = set(namelist)
                     root_safe_members = root_safe_module_class_members(classifier)
@@ -234,9 +238,13 @@ def root_safe_module_class_members(classifier):
                         raise Exception(
                             "root-safe module classes missing from aggregator: %s" %
                             ", ".join(missing_members))
+                    new_root_safe_members = (
+                        root_safe_members - promoted_root_safe_members)
                     zip_handle.extractall(
                         path=top_dist_jar_dir,
-                        members=[name for name in namelist if name in root_safe_members])
+                        members=[name for name in namelist
+                                 if name in new_root_safe_members])
+                    promoted_root_safe_members.update(root_safe_members)
                 # TODO deprecate
                 namelist = zip_handle.namelist()
                 glob_list = from_single_shim_or_each if bv == buildver_list[0] else from_each

dist/keep-in-spark-shim-dirs.txt

@@ -53,3 +53,12 @@ org/apache/spark/sql/rapids/execution/GpuBroadcastExchangeExec*.class
 # exchange execs. Keep the rule and helpers in the selected shim loader.
 com/nvidia/spark/rapids/GpuTransitionOverrides.class
 com/nvidia/spark/rapids/GpuTransitionOverrides$$*.class
+
+# Iceberg implementation classes compile against Spark- and Iceberg-version-specific
+# APIs. Root-selected Iceberg classes are excluded by binary-dedupe.sh; keep every
+# other Iceberg class in the selected shim loader instead of spark-shared.
+com/nvidia/spark/rapids/*Iceberg*.class
+com/nvidia/spark/rapids/fileio/iceberg/*.class
+com/nvidia/spark/rapids/iceberg/*.class
+org/apache/iceberg/*.class
+org/apache/spark/sql/rapids/*Iceberg*.class

dist/scripts/binary-dedupe.sh

@@ -182,6 +182,12 @@ function filter_keep_in_spark_shim_dirs() {
     path_without_leading_slash="${class_resource#/}"
     class_file="${path_without_leading_slash#*/}"
     if keep_in_spark_shim_dirs "$class_file"; then
+      # Root-layout classes were explicitly selected by the packager and must
+      # remain eligible for de-duplication so the later root promotion can
+      # remove their shim copies.
+      if [[ -f "./parallel-world/$class_file" ]]; then
+        echo "$class_resource"
+      fi
       continue
     fi
     echo "$class_resource"

dist/scripts/build-unshim-parallel-world.py

@@ -202,6 +202,10 @@ def copy_and_extract_jars(
     cache_root = target_dir / "unshim-parallel-world-cache"
     sorted_buildvers = sorted(buildvers, reverse=True)
     root_buildver = sorted_buildvers[0]
+    # Classifiers are processed newest-first. Let older classifiers contribute
+    # conditional root-safe classes that are absent from newer classifiers,
+    # without overwriting a newer implementation of the same class path.
+    promoted_root_safe_members = set()
 
     for buildver in sorted_buildvers:
         classifier = "spark%s" % buildver
@@ -216,7 +220,7 @@ def copy_and_extract_jars(
             link_tree_contents(contents_dir, parallel_world / classifier)
             if buildver == root_buildver and artifact == "sql-plugin-api":
                 link_tree_contents(contents_dir, parallel_world)
-            if buildver == root_buildver and artifact == "aggregator":
+            if artifact == "aggregator":
                 root_safe_members = root_safe_module_class_members(
                     base_dir,
                     scala_binary_version,
@@ -228,7 +232,11 @@ def copy_and_extract_jars(
                     raise RuntimeError(
                         "root-safe module classes missing from aggregator: %s" %
                         ", ".join(missing_members))
-                link_members(contents_dir, parallel_world, sorted(root_safe_members))
+                new_root_safe_members = (
+                    root_safe_members - promoted_root_safe_members)
+                link_members(contents_dir, parallel_world,
+                             sorted(new_root_safe_members))
+                promoted_root_safe_members.update(root_safe_members)
 
             patterns = from_each
             if buildver == root_buildver:

dist/unshimmed-common-from-single-shim.txt

@@ -11,13 +11,6 @@ com/nvidia/spark/rapids/optimizer/SQLOptimizerPlugin*
 com/nvidia/spark/rapids/ShimLoaderTemp*
 com/nvidia/spark/rapids/SparkShims*
 com/nvidia/shaded/spark/com/google/flatbuffers/*.class
-com/nvidia/spark/rapids/iceberg/parquet/converter/FromIcebergShaded*.class
-com/nvidia/spark/rapids/iceberg/parquet/converter/ToIcebergShaded*.class
-com/nvidia/spark/rapids/iceberg/spark/RapidsSparkCatalog.class
-com/nvidia/spark/rapids/iceberg/spark/RapidsSparkSessionCatalog.class
-com/nvidia/spark/rapids/iceberg/spark/source/RapidsSparkTable.class
-org/apache/iceberg/spark/source/GpuBaseReader.class
-org/apache/iceberg/spark/source/GpuSparkPlanningUtil.class
 org/apache/spark/sql/rapids/AdaptiveSparkPlanHelperShim*
 org/apache/spark/sql/rapids/ExecutionPlanCaptureCallback*
 rapids/*.py

iceberg-common/pom.xml

@@ -37,6 +37,14 @@
         <rapids.shim.jar.phase>package</rapids.shim.jar.phase>
         <rapids.shimplify.skip>false</rapids.shimplify.skip>
         <maven.scaladoc.skip>true</maven.scaladoc.skip>
+        <!--
+        Spark 4.x catalyst class files carry @Deprecated(since = "...")
+        annotations. Reading them under a Java 8 target makes javac emit
+        classfile warnings, which the project's -Werror promotes to an error.
+        Keep source warnings enabled while excluding dependency classfile
+        warnings for the conditional Java helpers in this module.
+        -->
+        <scala.javac.args>-Xlint:all,-serial,-path,-try,-processing,-options,-classfile|-Werror</scala.javac.args>
     </properties>
 
     <dependencies>

iceberg/common/src/main/java/com/nvidia/spark/rapids/iceberg/spark/RapidsSparkCatalog.java → iceberg-common/src/main/spark350/java/com/nvidia/spark/rapids/iceberg/spark/RapidsSparkCatalog.java

@@ -14,6 +14,27 @@
  * limitations under the License.
  */
 
+/*** spark-rapids-shim-json-lines
+{"spark": "350"}
+{"spark": "351"}
+{"spark": "352"}
+{"spark": "353"}
+{"spark": "354"}
+{"spark": "355"}
+{"spark": "356"}
+{"spark": "357"}
+{"spark": "358"}
+{"spark": "359"}
+{"spark": "400"}
+{"spark": "401"}
+{"spark": "402"}
+{"spark": "403"}
+{"spark": "404"}
+{"spark": "411"}
+{"spark": "412"}
+{"spark": "413"}
+spark-rapids-shim-json-lines ***/
+
 package com.nvidia.spark.rapids.iceberg.spark;
 
 import org.apache.iceberg.spark.SparkCatalog;

iceberg/common/src/main/java/com/nvidia/spark/rapids/iceberg/spark/RapidsSparkSessionCatalog.java → iceberg-common/src/main/spark350/java/com/nvidia/spark/rapids/iceberg/spark/RapidsSparkSessionCatalog.java

@@ -14,6 +14,27 @@
  * limitations under the License.
  */
 
+/*** spark-rapids-shim-json-lines
+{"spark": "350"}
+{"spark": "351"}
+{"spark": "352"}
+{"spark": "353"}
+{"spark": "354"}
+{"spark": "355"}
+{"spark": "356"}
+{"spark": "357"}
+{"spark": "358"}
+{"spark": "359"}
+{"spark": "400"}
+{"spark": "401"}
+{"spark": "402"}
+{"spark": "403"}
+{"spark": "404"}
+{"spark": "411"}
+{"spark": "412"}
+{"spark": "413"}
+spark-rapids-shim-json-lines ***/
+
 package com.nvidia.spark.rapids.iceberg.spark;
 
 import com.nvidia.spark.rapids.iceberg.spark.source.RapidsSparkTable;

iceberg/common/src/main/java/com/nvidia/spark/rapids/iceberg/spark/source/RapidsSparkTable.java → iceberg-common/src/main/spark350/java/com/nvidia/spark/rapids/iceberg/spark/source/RapidsSparkTable.java

@@ -14,6 +14,27 @@
  * limitations under the License.
  */
 
+/*** spark-rapids-shim-json-lines
+{"spark": "350"}
+{"spark": "351"}
+{"spark": "352"}
+{"spark": "353"}
+{"spark": "354"}
+{"spark": "355"}
+{"spark": "356"}
+{"spark": "357"}
+{"spark": "358"}
+{"spark": "359"}
+{"spark": "400"}
+{"spark": "401"}
+{"spark": "402"}
+{"spark": "403"}
+{"spark": "404"}
+{"spark": "411"}
+{"spark": "412"}
+{"spark": "413"}
+spark-rapids-shim-json-lines ***/
+
 package com.nvidia.spark.rapids.iceberg.spark.source;
 
 import java.util.Arrays;

iceberg/iceberg-1-6-x/src/main/java/org/apache/iceberg/spark/source/GpuBaseReader.java → iceberg-common/src/main/spark350/java/org/apache/iceberg/spark/source/GpuBaseReader.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2025, NVIDIA CORPORATION.
+ * Copyright (c) 2025-2026, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,6 +14,13 @@
  * limitations under the License.
  */
 
+/*** spark-rapids-shim-json-lines
+{"spark": "350"}
+{"spark": "351"}
+{"spark": "352"}
+{"spark": "353"}
+spark-rapids-shim-json-lines ***/
+
 package org.apache.iceberg.spark.source;
 
 import org.apache.iceberg.types.Type;

iceberg/common/src/main/java/org/apache/iceberg/spark/source/GpuSparkPlanningUtil.java → iceberg-common/src/main/spark350/java/org/apache/iceberg/spark/source/GpuSparkPlanningUtil.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2025, NVIDIA CORPORATION.
+ * Copyright (c) 2025-2026, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,6 +14,27 @@
  * limitations under the License.
  */
 
+/*** spark-rapids-shim-json-lines
+{"spark": "350"}
+{"spark": "351"}
+{"spark": "352"}
+{"spark": "353"}
+{"spark": "354"}
+{"spark": "355"}
+{"spark": "356"}
+{"spark": "357"}
+{"spark": "358"}
+{"spark": "359"}
+{"spark": "400"}
+{"spark": "401"}
+{"spark": "402"}
+{"spark": "403"}
+{"spark": "404"}
+{"spark": "411"}
+{"spark": "412"}
+{"spark": "413"}
+spark-rapids-shim-json-lines ***/
+
 package org.apache.iceberg.spark.source;
 
 import org.apache.iceberg.ScanTaskGroup;

scala2.13/iceberg-common/pom.xml

@@ -37,6 +37,14 @@
         <rapids.shim.jar.phase>package</rapids.shim.jar.phase>
         <rapids.shimplify.skip>false</rapids.shimplify.skip>
         <maven.scaladoc.skip>true</maven.scaladoc.skip>
+        <!--
+        Spark 4.x catalyst class files carry @Deprecated(since = "...")
+        annotations. Reading them under a Java 8 target makes javac emit
+        classfile warnings, which the project's -Werror promotes to an error.
+        Keep source warnings enabled while excluding dependency classfile
+        warnings for the conditional Java helpers in this module.
+        -->
+        <scala.javac.args>-Xlint:all,-serial,-path,-try,-processing,-options,-classfile|-Werror</scala.javac.args>
     </properties>
 
     <dependencies>