Support chunked libcudf extraction for native UDF builds

.github/workflows/native-lib-extraction-test.yml

@@ -0,0 +1,44 @@
+# Copyright (c) 2026, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Native library extraction tests
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "examples/UDF-Examples/RAPIDS-accelerated-UDFs/extract-native-libs.py"
+      - "examples/UDF-Examples/RAPIDS-accelerated-UDFs/test_extract_native_libs.py"
+      - ".github/workflows/native-lib-extraction-test.yml"
+  pull_request:
+    paths:
+      - "examples/UDF-Examples/RAPIDS-accelerated-UDFs/extract-native-libs.py"
+      - "examples/UDF-Examples/RAPIDS-accelerated-UDFs/test_extract_native_libs.py"
+      - ".github/workflows/native-lib-extraction-test.yml"
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run native library extraction tests
+        run: |
+          python3 -m unittest discover \
+            -s examples/UDF-Examples/RAPIDS-accelerated-UDFs \
+            -p "test_extract_native_libs.py" \
+            -v

examples/UDF-Examples/RAPIDS-accelerated-UDFs/README.md

@@ -145,7 +145,8 @@ mvn clean package -Pudf-native-examples
 ```
 
 The build will automatically:
-- Extract `libcudf.so` from the rapids-4-spark jar
+- Extract `libcudf.so` from older rapids-4-spark jars or reconstruct it from
+  the chunk-manifest representation used by newer jars
 - Read the embedded `spark-rapids-jni` and `cudf-java` version metadata from the jar
 - Download the matching `spark-rapids-jni` `cudf-pins` files
 - Clone the cuDF repository at the revision recorded in the jar

examples/UDF-Examples/RAPIDS-accelerated-UDFs/extract-cudf-libs.sh

@@ -149,103 +149,25 @@ else
     exit 1
 fi
 
-# Extract libcudf.so and dependencies
+# Extract or reconstruct libcudf.so and dependencies
 echo "Extracting native libraries from jar..."
 echo "  Jar: $JAR_PATH"
-echo "  Looking for: */libcudf.so*, */libnvcomp.so*"
+echo "  Supports conventional and chunked native library entries"
 
-# Use unzip without -q to capture output, but redirect to log for debugging
-UNZIP_OUTPUT=$(unzip -o "$JAR_PATH" "*/libcudf.so*" "*/libnvcomp.so*" -d "$TARGET_DIR/temp" 2>&1)
-UNZIP_EXIT_CODE=$?
-
-# Check unzip exit code
-if [ $UNZIP_EXIT_CODE -ne 0 ]; then
-    echo "ERROR: Failed to extract libraries from jar" >&2
-    echo "unzip exit code: $UNZIP_EXIT_CODE" >&2
-
-    # Provide helpful diagnostics
-    case $UNZIP_EXIT_CODE in
-        11)
-            echo "Reason: No matching files found in jar" >&2
-            echo "" >&2
-            echo "The jar may not contain native libraries for your platform." >&2
-            echo "Expected patterns: */libcudf.so*, */libnvcomp.so*" >&2
-            echo "" >&2
-            echo "Listing jar contents:" >&2
-            unzip -l "$JAR_PATH" | grep -E '\.(so|dylib|dll)' || echo "  No native libraries found" >&2
-            ;;
-        *)
-            echo "Reason: unzip command failed" >&2
-            echo "Output: $UNZIP_OUTPUT" >&2
-            ;;
-    esac
-
-    echo "" >&2
-    echo "Falling back to source build..." >&2
-    exit 1
-fi
-
-# Verify that we actually extracted some files
-EXTRACTED_COUNT=$(find "$TARGET_DIR/temp" -name "*.so*" 2>/dev/null | wc -l)
-echo "Extracted $EXTRACTED_COUNT library file(s)"
-
-if [ "$EXTRACTED_COUNT" -eq 0 ]; then
-    echo "ERROR: No library files were extracted from jar" >&2
-    echo "This usually means the jar doesn't contain native libraries." >&2
-    echo "" >&2
-    echo "Listing jar contents:" >&2
-    unzip -l "$JAR_PATH" | head -20 >&2
+if ! command -v python3 >/dev/null 2>&1; then
+    echo "ERROR: python3 is required to extract native libraries" >&2
     exit 1
 fi
 
-# Move libraries to native-deps directory, detecting conflicts
-echo "Moving extracted libraries..."
-CONFLICT_COUNT=0
-
-# Use process substitution to avoid subshell issues
-while IFS= read -r source_file; do
-    filename=$(basename "$source_file")
-    dest_file="$NATIVE_DEPS_DIR/$filename"
-
-    if [ -f "$dest_file" ]; then
-        # File already exists - check if it's the same
-        if ! cmp -s "$source_file" "$dest_file"; then
-            echo "WARNING: Conflicting library detected: $filename" >&2
-            echo "  Existing: $dest_file" >&2
-            echo "  New:      $source_file" >&2
-            echo "  Keeping existing file, skipping new one" >&2
-            CONFLICT_COUNT=$((CONFLICT_COUNT + 1))
-        fi
-        # Remove the duplicate source file
-        rm -f "$source_file"
-    else
-        # No conflict, move the file
-        mv "$source_file" "$dest_file"
-    fi
-done < <(find "$TARGET_DIR/temp" -name "*.so*")
-
-if [ "$CONFLICT_COUNT" -gt 0 ]; then
-    echo "WARNING: $CONFLICT_COUNT library file(s) had conflicts. Review the warnings above." >&2
-fi
-
-rm -rf "$TARGET_DIR/temp"
+python3 "$SCRIPT_DIR/extract-native-libs.py" "$JAR_PATH" "$NATIVE_DEPS_DIR"
 
-# Verify that libcudf.so was successfully moved to final location
+# Verify that libcudf.so was successfully written to the final location
 if [ ! -f "$NATIVE_DEPS_DIR/libcudf.so" ]; then
     echo "ERROR: libcudf.so not found in $NATIVE_DEPS_DIR" >&2
-    echo "" >&2
-    echo "This could mean:" >&2
-    echo "  1. The jar didn't contain libcudf.so" >&2
-    echo "  2. Extraction succeeded but moving files failed" >&2
-    echo "  3. Wrong architecture (jar might be for a different platform)" >&2
-    echo "" >&2
-    echo "Contents of $NATIVE_DEPS_DIR:" >&2
-    ls -lh "$NATIVE_DEPS_DIR" >&2 || echo "  Directory is empty or doesn't exist" >&2
     exit 1
 fi
 
 echo "✓ Successfully extracted libraries to: $NATIVE_DEPS_DIR"
-ls -lh "$NATIVE_DEPS_DIR"
 
 PINS_DIR="$TARGET_DIR/cudf-pins"
 PINS_PROPERTIES="$TARGET_DIR/jar-native-deps.properties"