diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ac8a3f63300..7b7fac76c6c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -8,6 +8,13 @@ /project/ @tdas /version.sbt @tdas +# Python client, examples, and tooling +/python/ @tdas + +# Iceberg / UniForm integration and shaded Iceberg sources +/iceberg/ @tdas +/icebergShaded/ @tdas + # Spark V2 and Unified modules /spark/v2/ @tdas @huan233usc @TimothyW553 @raveeram-db @murali-db /spark-unified/ @tdas @huan233usc @TimothyW553 @raveeram-db @murali-db diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 207817a51bd..8bee17570e0 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -13,21 +13,21 @@ jobs: name: "Build Test" runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - name: Install Java 17 - uses: actions/setup-java@v3 + uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1 with: distribution: "zulu" java-version: "17" - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4.9.1 with: python-version: '3.9' - name: Cache Scala, SBT - uses: actions/cache@v3 + uses: actions/cache@6f8efc29b200d32929f49075959781ed54ec270c # v3.5.0 with: path: | ~/.sbt diff --git a/.github/workflows/iceberg_test.yaml b/.github/workflows/disabled_iceberg_test.yaml similarity index 75% rename from .github/workflows/iceberg_test.yaml rename to .github/workflows/disabled_iceberg_test.yaml index 9750ca7d6dc..3f6b19fe81c 100644 --- a/.github/workflows/iceberg_test.yaml +++ b/.github/workflows/disabled_iceberg_test.yaml @@ -1,15 +1,13 @@ -name: "Delta Iceberg Latest" +name: "Delta Iceberg Latest [DISABLED]" +# SECURITY: All Python/PySpark workflows disabled due to active supply chain attack +# targeting OSS package ecosystems (PyPI). C2 domains: models.litellm.cloud, checkmarx.zone +# Date disabled: 2026-03-25 +# To re-enable: remove 'if: false' from all jobs and restore original triggers on: - push: - paths-ignore: - - '**.md' - - '**.txt' - pull_request: - paths-ignore: - - '**.md' - - '**.txt' + workflow_dispatch: # manual-only, auto triggers removed jobs: test: + if: false # SECURITY: disabled - supply chain attack mitigation name: "DIL: Scala ${{ matrix.scala }}" runs-on: ubuntu-24.04 strategy: @@ -19,14 +17,14 @@ jobs: env: SCALA_VERSION: ${{ matrix.scala }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - name: install java - uses: actions/setup-java@v3 + uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1 with: distribution: "zulu" java-version: "17" - name: Cache Scala, SBT - uses: actions/cache@v3 + uses: actions/cache@6f8efc29b200d32929f49075959781ed54ec270c # v3.5.0 with: path: | ~/.sbt diff --git a/.github/workflows/publish_docs.yaml b/.github/workflows/disabled_publish_docs.yaml similarity index 72% rename from .github/workflows/publish_docs.yaml rename to .github/workflows/disabled_publish_docs.yaml index 15b56750855..b5c2d03e600 100644 --- a/.github/workflows/publish_docs.yaml +++ b/.github/workflows/disabled_publish_docs.yaml @@ -1,18 +1,15 @@ -name: Publish Docs +name: "Publish Docs [DISABLED]" +# SECURITY: All Python/PySpark workflows disabled due to active supply chain attack +# targeting OSS package ecosystems (PyPI). C2 domains: models.litellm.cloud, checkmarx.zone +# Date disabled: 2026-03-25 +# To re-enable: remove 'if: false' from all jobs and restore original triggers on: - push: - branches: - - master - paths: - - docs/** - release: - types: - - published - workflow_dispatch: + workflow_dispatch: # manual-only, auto triggers removed jobs: build_api_docs: + if: false # SECURITY: disabled - supply chain attack mitigation name: Build API docs (${{ matrix.version.name }}) runs-on: ubuntu-latest @@ -29,18 +26,18 @@ jobs: out_dir: docs/apis/_site/api steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: repository: delta-io/delta ref: ${{ matrix.version.ref }} - - uses: actions/setup-java@v3 + - uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1 with: distribution: "zulu" java-version: ${{ matrix.version.java }} - name: Setup python environment - uses: conda-incubator/setup-miniconda@v3 + uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3.3.0 with: activate-environment: delta_docs environment-file: docs/environment.yml @@ -71,25 +68,26 @@ jobs: done - name: Upload artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: ${{ github.run_id }}-apidocs-${{ matrix.version.name }} path: ${{ matrix.version.out_dir }} build_site: + if: false # SECURITY: disabled - supply chain attack mitigation name: Build site needs: build_api_docs runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Install Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: docs/.nvmrc - - uses: pnpm/action-setup@v4 + - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 name: Install pnpm with: package_json_file: docs/package.json @@ -99,7 +97,7 @@ jobs: working-directory: docs - name: Download API docs artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: pattern: ${{ github.run_id }}-apidocs-* path: docs/public/api diff --git a/.github/workflows/spark_python_test.yaml b/.github/workflows/disabled_spark_python_test.yaml similarity index 86% rename from .github/workflows/spark_python_test.yaml rename to .github/workflows/disabled_spark_python_test.yaml index 7fd21e9e37a..b57d068cac4 100644 --- a/.github/workflows/spark_python_test.yaml +++ b/.github/workflows/disabled_spark_python_test.yaml @@ -1,25 +1,23 @@ -name: "Delta Spark Python" +name: "Delta Spark Python [DISABLED]" +# SECURITY: All Python/PySpark workflows disabled due to active supply chain attack +# targeting OSS package ecosystems (PyPI). C2 domains: models.litellm.cloud, checkmarx.zone +# Date disabled: 2026-03-25 +# To re-enable: remove 'if: false' from all jobs and restore original triggers on: - push: - paths-ignore: - - '**.md' - - '**.txt' - pull_request: - paths-ignore: - - '**.md' - - '**.txt' + workflow_dispatch: # manual-only, auto triggers removed jobs: # Generate Spark versions matrix from CrossSparkVersions.scala # This workflow tests against released versions only (no snapshots) generate-matrix: + if: false # SECURITY: disabled - supply chain attack mitigation name: "Generate Released Spark Versions Matrix" runs-on: ubuntu-24.04 outputs: spark_versions: ${{ steps.generate.outputs.spark_versions }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - name: install java - uses: actions/setup-java@v3 + uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1 with: distribution: "zulu" java-version: "17" @@ -32,6 +30,7 @@ jobs: echo "Generated released Spark versions: $SPARK_VERSIONS" test: + if: false # SECURITY: disabled - supply chain attack mitigation name: "DSP (${{ matrix.spark_version }})" runs-on: ubuntu-24.04 needs: generate-matrix @@ -45,7 +44,7 @@ jobs: SCALA_VERSION: ${{ matrix.scala }} SPARK_VERSION: ${{ matrix.spark_version }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - name: Get Spark version details id: spark-details run: | @@ -56,12 +55,12 @@ jobs: echo "spark_full_version=$FULL_VERSION" >> $GITHUB_OUTPUT echo "Using JVM $JVM_VERSION for Spark ${{ matrix.spark_version }} (full: $FULL_VERSION)" - name: install java - uses: actions/setup-java@v3 + uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1 with: distribution: "zulu" java-version: ${{ steps.spark-details.outputs.jvm_version }} - name: Cache Scala, SBT - uses: actions/cache@v3 + uses: actions/cache@6f8efc29b200d32929f49075959781ed54ec270c # v3.5.0 with: path: | ~/.sbt diff --git a/.github/workflows/flink_test.yaml b/.github/workflows/flink_test.yaml index 487afd8976e..71fd9df2111 100644 --- a/.github/workflows/flink_test.yaml +++ b/.github/workflows/flink_test.yaml @@ -27,16 +27,16 @@ jobs: echo "Runner OS: ${{ runner.os }}" echo "Runner arch: ${{ runner.arch }}" - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 # Run unit tests with JDK 17. These unit tests depend on Spark, and Spark 4.0+ is JDK 17. - name: install java - uses: actions/setup-java@v4 + uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4.8.0 with: distribution: "zulu" java-version: "17" - name: Cache SBT and dependencies id: cache-sbt - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | ~/.sbt diff --git a/.github/workflows/kernel_docs.yaml b/.github/workflows/kernel_docs.yaml index 885c2c3047c..f82db0b46eb 100644 --- a/.github/workflows/kernel_docs.yaml +++ b/.github/workflows/kernel_docs.yaml @@ -26,9 +26,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - name: install java - uses: actions/setup-java@v3 + uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1 with: distribution: "zulu" java-version: "11" @@ -40,12 +40,12 @@ jobs: cp -r kernel/kernel-api/target/javaunidoc/. kernel/docs/snapshot/kernel-api/java/ cp -r kernel/kernel-defaults/target/javaunidoc/. kernel/docs/snapshot/kernel-defaults/java/ - name: Setup Pages - uses: actions/configure-pages@v3 + uses: actions/configure-pages@b8130d9ab958b325bbde9786d62f2c97a9885a0e # v3.0.7 - name: Upload artifact - uses: actions/upload-pages-artifact@v1 + uses: actions/upload-pages-artifact@84bb4cd4b733d5c320c9c9cfbc354937524f4d64 # v1.0.10 with: # Upload kernel docs path: kernel/docs - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v2 + uses: actions/deploy-pages@de14547edc9944350dc0481aa5b7afb08e75f254 # v2.0.5 diff --git a/.github/workflows/kernel_test.yaml b/.github/workflows/kernel_test.yaml index e0e2f5bdba7..0e104311d70 100644 --- a/.github/workflows/kernel_test.yaml +++ b/.github/workflows/kernel_test.yaml @@ -44,16 +44,16 @@ jobs: echo "Runner OS: ${{ runner.os }}" echo "Runner arch: ${{ runner.arch }}" - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 # Run unit tests with JDK 17. These unit tests depend on Spark, and Spark 4.0+ is JDK 17. - name: install java - uses: actions/setup-java@v4 + uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4.8.0 with: distribution: "zulu" java-version: "17" - name: Cache SBT and dependencies id: cache-sbt - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | ~/.sbt @@ -76,10 +76,10 @@ jobs: name: "DK: Integration" runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 # Run integration tests with JDK 11, as they have no Spark dependency - name: install java - uses: actions/setup-java@v3 + uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1 with: distribution: "zulu" java-version: "11" diff --git a/.github/workflows/kernel_unitycatalog_test.yaml b/.github/workflows/kernel_unitycatalog_test.yaml index f71722112d5..72830cfa3cb 100644 --- a/.github/workflows/kernel_unitycatalog_test.yaml +++ b/.github/workflows/kernel_unitycatalog_test.yaml @@ -25,9 +25,9 @@ jobs: env: SCALA_VERSION: 2.13.16 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - name: install java - uses: actions/setup-java@v3 + uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1 with: distribution: "zulu" java-version: "17" diff --git a/.github/workflows/spark_examples_test.yaml b/.github/workflows/spark_examples_test.yaml index a7cf25c5011..3748f91c661 100644 --- a/.github/workflows/spark_examples_test.yaml +++ b/.github/workflows/spark_examples_test.yaml @@ -17,9 +17,9 @@ jobs: outputs: spark_versions: ${{ steps.generate.outputs.spark_versions }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - name: install java - uses: actions/setup-java@v3 + uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1 with: distribution: "zulu" java-version: "17" @@ -44,7 +44,7 @@ jobs: env: SCALA_VERSION: ${{ matrix.scala }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - name: Get Spark version details id: spark-details run: | @@ -59,12 +59,12 @@ jobs: echo "spark_full_version=$SPARK_FULL_VERSION" >> $GITHUB_OUTPUT echo "Using JVM $JVM_VERSION for Spark $SPARK_FULL_VERSION, package suffix: '$SPARK_PACKAGE_SUFFIX', support iceberg: '$SUPPORT_ICEBERG'" - name: install java - uses: actions/setup-java@v3 + uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1 with: distribution: "zulu" java-version: ${{ steps.spark-details.outputs.jvm_version }} - name: Cache Scala, SBT - uses: actions/cache@v3 + uses: actions/cache@6f8efc29b200d32929f49075959781ed54ec270c # v3.5.0 with: path: | ~/.sbt diff --git a/.github/workflows/spark_test.yaml b/.github/workflows/spark_test.yaml index 20bb9ad01a5..8f846c6b930 100644 --- a/.github/workflows/spark_test.yaml +++ b/.github/workflows/spark_test.yaml @@ -17,9 +17,9 @@ jobs: outputs: spark_versions: ${{ steps.generate.outputs.spark_versions }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - name: install java - uses: actions/setup-java@v3 + uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1 with: distribution: "zulu" java-version: "17" @@ -51,7 +51,7 @@ jobs: # Important: This must be the same as the length of shards in matrix NUM_SHARDS: 8 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - name: Get Spark version details id: spark-details run: | @@ -60,12 +60,12 @@ jobs: echo "jvm_version=$JVM_VERSION" >> $GITHUB_OUTPUT echo "Using JVM version: $JVM_VERSION for Spark ${{ matrix.spark_version }}" - name: install java - uses: actions/setup-java@v3 + uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1 with: distribution: "zulu" java-version: ${{ steps.spark-details.outputs.jvm_version }} - name: Cache Scala, SBT - uses: actions/cache@v3 + uses: actions/cache@6f8efc29b200d32929f49075959781ed54ec270c # v3.5.0 with: path: | ~/.sbt @@ -76,62 +76,17 @@ jobs: # just use the cache. The cache is immutable so we need to use a new key when trying to # cache new stuff. key: delta-sbt-cache-spark${{ matrix.spark_version }}-scala${{ matrix.scala }} - - name: Install Job dependencies - run: | - sudo apt-get update - sudo apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev python3-openssl git - sudo apt install libedit-dev - curl -LO https://github.com/bufbuild/buf/releases/download/v1.28.1/buf-Linux-x86_64.tar.gz - mkdir -p ~/buf - tar -xvzf buf-Linux-x86_64.tar.gz -C ~/buf --strip-components 1 - rm buf-Linux-x86_64.tar.gz - sudo apt install python3-pip --fix-missing - sudo pip3 install pipenv==2024.4.1 - curl https://pyenv.run | bash - export PATH="~/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - eval "$(pyenv virtualenv-init -)" - pyenv install 3.9 - pyenv global system 3.9 - pipenv --python 3.9 install - # Update the pip version to 24.0. By default `pyenv.run` installs the latest pip version - # available. From version 24.1, `pip` doesn't allow installing python packages - # with version string containing `-`. In Delta-Spark case, the pypi package generated has - # `-SNAPSHOT` in version (e.g. `3.3.0-SNAPSHOT`) as the version is picked up from - # the`version.sbt` file. - pipenv run pip install pip==24.0 setuptools==69.5.1 wheel==0.43.0 - pipenv run pip install flake8==3.9.0 - pipenv run pip install black==23.12.1 - pipenv run pip install importlib_metadata==3.10.0 - pipenv run pip install mypy==1.8.0 - pipenv run pip install mypy-protobuf==3.3.0 - pipenv run pip install cryptography==37.0.4 - pipenv run pip install twine==4.0.1 - pipenv run pip install wheel==0.33.4 - pipenv run pip install setuptools==41.1.0 - pipenv run pip install pydocstyle==3.0.0 - pipenv run pip install pandas==2.2.0 - pipenv run pip install pyarrow==11.0.0 - pipenv run pip install pypandoc==1.3.3 - pipenv run pip install numpy==1.22.4 - pipenv run pip install grpcio==1.67.0 - pipenv run pip install grpcio-status==1.67.0 - pipenv run pip install googleapis-common-protos==1.65.0 - pipenv run pip install protobuf==5.29.1 - pipenv run pip install googleapis-common-protos-stubs==2.2.0 - pipenv run pip install grpc-stubs==1.24.11 - name: Scala structured logging style check run: | if [ -f ./dev/spark_structured_logging_style.py ]; then python3 ./dev/spark_structured_logging_style.py fi - name: Run Scala/Java tests - # when changing TEST_PARALLELISM_COUNT make sure to also change it in spark_python_test.yaml run: | - TEST_PARALLELISM_COUNT=4 pipenv run python run-tests.py --group spark --shard ${{ matrix.shard }} --spark-version ${{ matrix.spark_version }} + TEST_PARALLELISM_COUNT=4 python3 run-tests.py --group spark --shard ${{ matrix.shard }} --spark-version ${{ matrix.spark_version }} - name: Upload test reports if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: test-reports-spark${{ matrix.spark_version }}-shard${{ matrix.shard }} path: "**/target/test-reports/*.xml" diff --git a/.github/workflows/unidoc.yaml b/.github/workflows/unidoc.yaml index 735bdf806d4..c9c5cc525db 100644 --- a/.github/workflows/unidoc.yaml +++ b/.github/workflows/unidoc.yaml @@ -10,10 +10,10 @@ scala: [2.13.16] steps: - name: install java - uses: actions/setup-java@v3 + uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1 with: distribution: "zulu" java-version: "17" - - uses: actions/checkout@v3 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - name: generate unidoc run: build/sbt "++ ${{ matrix.scala }}" unidoc diff --git a/PROTOCOL.md b/PROTOCOL.md index 1c989a18948..c9015a57f06 100644 --- a/PROTOCOL.md +++ b/PROTOCOL.md @@ -893,7 +893,7 @@ A given snapshot of the table can be computed by replaying the events committed - A single `protocol` action - A single `metaData` action - A collection of `txn` actions with unique `appId`s - - A collection of `domainMetadata` actions with unique `domain`s. + - A collection of `domainMetadata` actions with unique `domain`s, excluding tombstones (i.e. actions with `removed=true`). - A collection of `add` actions with unique path keys, corresponding to the newest (path, deletionVector.uniqueId) pair encountered for each path. - A collection of `remove` actions with unique `(path, deletionVector.uniqueId)` keys. The intersection of the primary keys in the `add` collection and `remove` collection must be empty. That means a logical file cannot exist in both the `remove` and `add` collections at the same time; however, the same *data file* can exist with *different* DVs in the `remove` collection, as logically they represent different content. The `remove` actions act as _tombstones_, and only exist for the benefit of the VACUUM command. Snapshot reads only return `add` actions on the read path. @@ -2090,7 +2090,7 @@ Each row in the checkpoint corresponds to a single action. The checkpoint **must * Files that have been [added](#Add-File-and-Remove-File) and not yet removed * Files that were recently [removed](#Add-File-and-Remove-File) and have not yet expired * [Transaction identifiers](#Transaction-Identifiers) - * [Domain Metadata](#Domain-Metadata) + * [Domain Metadata](#Domain-Metadata) that have not been removed (i.e. excluding tombstones with `removed=true`) * [Checkpoint Metadata](#checkpoint-metadata) - Requires [V2 checkpoints](#v2-spec) * [Sidecar File](#sidecar-files) - Requires [V2 checkpoints](#v2-spec) diff --git a/README.md b/README.md index ebd2d24b76a..5a3c8ceaa50 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,7 @@ We also adhere to the [Delta Lake Code of Conduct](https://github.com/delta-io/d ## Building Delta Lake is compiled using [SBT](https://www.scala-sbt.org/1.x/docs/Command-Line-Reference.html). +Ensure that your Java version is at least 17 (you can verify with `java -version`). To compile, run diff --git a/build.sbt b/build.sbt index 14c113d59b1..8f4e27951fe 100644 --- a/build.sbt +++ b/build.sbt @@ -822,7 +822,7 @@ lazy val sharing = (project in file("sharing")) libraryDependencies ++= Seq( "org.apache.spark" %% "spark-sql" % sparkVersion.value % "provided", - "io.delta" %% "delta-sharing-client" % "1.3.9", + "io.delta" %% "delta-sharing-client" % "1.3.10", // Test deps "org.scalatest" %% "scalatest" % scalaTestVersion % "test", diff --git a/connectors/golden-tables/src/main/resources/golden/collations-preview-table/_delta_log/.00000000000000000000.json.crc b/connectors/golden-tables/src/main/resources/golden/collations-preview-table/_delta_log/.00000000000000000000.json.crc new file mode 100644 index 00000000000..69ac0d941a9 Binary files /dev/null and b/connectors/golden-tables/src/main/resources/golden/collations-preview-table/_delta_log/.00000000000000000000.json.crc differ diff --git a/connectors/golden-tables/src/main/resources/golden/collations-preview-table/_delta_log/00000000000000000000.crc b/connectors/golden-tables/src/main/resources/golden/collations-preview-table/_delta_log/00000000000000000000.crc new file mode 100644 index 00000000000..b499b99ebae --- /dev/null +++ b/connectors/golden-tables/src/main/resources/golden/collations-preview-table/_delta_log/00000000000000000000.crc @@ -0,0 +1 @@ +{"txnId":"6132e880-0f3a-4db4-b882-1da039bffbad","tableSizeBytes":0,"numFiles":0,"numMetadata":1,"numProtocol":1,"setTransactions":[],"domainMetadata":[],"metadata":{"id":"0eb3e007-b3cc-40e4-bca1-a5970d86b5a6","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"utf8_binary_col\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"utf8_lcase_col\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"__COLLATIONS\":{\"utf8_lcase_col\":\"spark.UTF8_LCASE\"}}},{\"name\":\"unicode_col\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"__COLLATIONS\":{\"unicode_col\":\"icu.UNICODE\"}}}]}","partitionColumns":[],"configuration":{},"createdTime":1773779518731},"protocol":{"minReaderVersion":1,"minWriterVersion":7,"writerFeatures":["domainMetadata","collations-preview","appendOnly","invariants"]},"histogramOpt":{"sortedBinBoundaries":[0,8192,16384,32768,65536,131072,262144,524288,1048576,2097152,4194304,8388608,12582912,16777216,20971520,25165824,29360128,33554432,37748736,41943040,50331648,58720256,67108864,75497472,83886080,92274688,100663296,109051904,117440512,125829120,130023424,134217728,138412032,142606336,146800640,150994944,167772160,184549376,201326592,218103808,234881024,251658240,268435456,285212672,301989888,318767104,335544320,352321536,369098752,385875968,402653184,419430400,436207616,452984832,469762048,486539264,503316480,520093696,536870912,553648128,570425344,587202560,603979776,671088640,738197504,805306368,872415232,939524096,1006632960,1073741824,1140850688,1207959552,1275068416,1342177280,1409286144,1476395008,1610612736,1744830464,1879048192,2013265920,2147483648,2415919104,2684354560,2952790016,3221225472,3489660928,3758096384,4026531840,4294967296,8589934592,17179869184,34359738368,68719476736,137438953472,274877906944],"fileCounts":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"totalBytes":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"allFiles":[]} diff --git a/connectors/golden-tables/src/main/resources/golden/collations-preview-table/_delta_log/00000000000000000000.json b/connectors/golden-tables/src/main/resources/golden/collations-preview-table/_delta_log/00000000000000000000.json new file mode 100644 index 00000000000..d83ea327864 --- /dev/null +++ b/connectors/golden-tables/src/main/resources/golden/collations-preview-table/_delta_log/00000000000000000000.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1773779521901,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","statsOnLoad":false,"partitionBy":"[]"},"isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{"numFiles":"0","numOutputRows":"0","numOutputBytes":"0"},"tags":{"noRowsCopied":"true","restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/","txnId":"6132e880-0f3a-4db4-b882-1da039bffbad"}} +{"metaData":{"id":"0eb3e007-b3cc-40e4-bca1-a5970d86b5a6","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"utf8_binary_col\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"utf8_lcase_col\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"__COLLATIONS\":{\"utf8_lcase_col\":\"spark.UTF8_LCASE\"}}},{\"name\":\"unicode_col\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"__COLLATIONS\":{\"unicode_col\":\"icu.UNICODE\"}}}]}","partitionColumns":[],"configuration":{},"createdTime":1773779518731}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":7,"writerFeatures":["domainMetadata","collations-preview","appendOnly","invariants"]}} diff --git a/connectors/golden-tables/src/main/resources/golden/collations-table/_delta_log/.00000000000000000000.json.crc b/connectors/golden-tables/src/main/resources/golden/collations-table/_delta_log/.00000000000000000000.json.crc new file mode 100644 index 00000000000..e62bc23a8ef Binary files /dev/null and b/connectors/golden-tables/src/main/resources/golden/collations-table/_delta_log/.00000000000000000000.json.crc differ diff --git a/connectors/golden-tables/src/main/resources/golden/collations-table/_delta_log/00000000000000000000.crc b/connectors/golden-tables/src/main/resources/golden/collations-table/_delta_log/00000000000000000000.crc new file mode 100644 index 00000000000..3fd0fbb8fd3 --- /dev/null +++ b/connectors/golden-tables/src/main/resources/golden/collations-table/_delta_log/00000000000000000000.crc @@ -0,0 +1 @@ +{"txnId":"534b1ff2-bfc1-4d6f-b435-08baef2d8dd0","tableSizeBytes":0,"numFiles":0,"numMetadata":1,"numProtocol":1,"setTransactions":[],"domainMetadata":[],"metadata":{"id":"dffc30c0-bb9a-4f95-91f9-3141788a8094","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"utf8_binary_col\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"utf8_lcase_col\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"__COLLATIONS\":{\"utf8_lcase_col\":\"spark.UTF8_LCASE\"}}},{\"name\":\"unicode_col\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"__COLLATIONS\":{\"unicode_col\":\"icu.UNICODE\"}}}]}","partitionColumns":[],"configuration":{},"createdTime":1773774637236},"protocol":{"minReaderVersion":1,"minWriterVersion":7,"writerFeatures":["domainMetadata","collations","appendOnly","invariants"]},"histogramOpt":{"sortedBinBoundaries":[0,8192,16384,32768,65536,131072,262144,524288,1048576,2097152,4194304,8388608,12582912,16777216,20971520,25165824,29360128,33554432,37748736,41943040,50331648,58720256,67108864,75497472,83886080,92274688,100663296,109051904,117440512,125829120,130023424,134217728,138412032,142606336,146800640,150994944,167772160,184549376,201326592,218103808,234881024,251658240,268435456,285212672,301989888,318767104,335544320,352321536,369098752,385875968,402653184,419430400,436207616,452984832,469762048,486539264,503316480,520093696,536870912,553648128,570425344,587202560,603979776,671088640,738197504,805306368,872415232,939524096,1006632960,1073741824,1140850688,1207959552,1275068416,1342177280,1409286144,1476395008,1610612736,1744830464,1879048192,2013265920,2147483648,2415919104,2684354560,2952790016,3221225472,3489660928,3758096384,4026531840,4294967296,8589934592,17179869184,34359738368,68719476736,137438953472,274877906944],"fileCounts":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"totalBytes":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"allFiles":[]} diff --git a/connectors/golden-tables/src/main/resources/golden/collations-table/_delta_log/00000000000000000000.json b/connectors/golden-tables/src/main/resources/golden/collations-table/_delta_log/00000000000000000000.json new file mode 100644 index 00000000000..7c2507b1d1b --- /dev/null +++ b/connectors/golden-tables/src/main/resources/golden/collations-table/_delta_log/00000000000000000000.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1773774663439,"operation":"CREATE TABLE","operationParameters":{"partitionBy":"[]","clusterBy":"[]","description":null,"isManaged":"false","properties":"{}","statsOnLoad":false},"isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{},"tags":{"restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/","txnId":"534b1ff2-bfc1-4d6f-b435-08baef2d8dd0"}} +{"metaData":{"id":"dffc30c0-bb9a-4f95-91f9-3141788a8094","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"utf8_binary_col\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"utf8_lcase_col\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"__COLLATIONS\":{\"utf8_lcase_col\":\"spark.UTF8_LCASE\"}}},{\"name\":\"unicode_col\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"__COLLATIONS\":{\"unicode_col\":\"icu.UNICODE\"}}}]}","partitionColumns":[],"configuration":{},"createdTime":1773774637236}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":7,"writerFeatures":["domainMetadata","collations","appendOnly","invariants"]}} diff --git a/iceberg/src/main/scala/org/apache/spark/sql/delta/IcebergFileManifest.scala b/iceberg/src/main/scala/org/apache/spark/sql/delta/IcebergFileManifest.scala index 9109c7d0d34..590348a622d 100644 --- a/iceberg/src/main/scala/org/apache/spark/sql/delta/IcebergFileManifest.scala +++ b/iceberg/src/main/scala/org/apache/spark/sql/delta/IcebergFileManifest.scala @@ -182,7 +182,11 @@ class IcebergFileManifest( val dataFiles = spark .createDataset(manifestFiles) - .flatMap(ManifestFiles.read(_, localTable.io()).asScala.map(new DataFileWrapper(_))) + .flatMap( + ManifestFiles.read(_, localTable.io(), localTable.specs()) + .asScala.map(new DataFileWrapper(_) + ) + ) dataFiles } diff --git a/iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergConversionTransaction.scala b/iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergConversionTransaction.scala index 71e69281d69..e18dd49faab 100644 --- a/iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergConversionTransaction.scala +++ b/iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergConversionTransaction.scala @@ -22,6 +22,7 @@ import java.util.function.Consumer import scala.collection.JavaConverters._ import scala.collection.mutable import scala.collection.mutable.ArrayBuffer +import scala.jdk.OptionConverters._ import scala.util.control.NonFatal import org.apache.spark.sql.delta.{DeltaFileProviderUtils, DummySnapshot, IcebergConstants, NoMapping, Snapshot} @@ -33,11 +34,12 @@ import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.commons.lang3.exception.ExceptionUtils import org.apache.hadoop.conf.Configuration -import shadedForDelta.org.apache.iceberg.{AppendFiles, DataFile, DeleteFiles, ExpireSnapshots, OverwriteFiles, PartitionSpec, PendingUpdate, RewriteFiles, Schema => IcebergSchema, Transaction => IcebergTransaction} +import shadedForDelta.org.apache.iceberg.{AppendFiles, BaseTransaction, DataFile, DeleteFiles, ExpireSnapshots, OverwriteFiles, PartitionSpec, PendingUpdate, RewriteFiles, Schema => IcebergSchema, TableMetadata, Transaction => IcebergTransaction} import shadedForDelta.org.apache.iceberg.MetadataUpdate import shadedForDelta.org.apache.iceberg.MetadataUpdate.{AddPartitionSpec, AddSchema} import shadedForDelta.org.apache.iceberg.mapping.MappingUtil import shadedForDelta.org.apache.iceberg.mapping.NameMappingParser +import shadedForDelta.org.apache.iceberg.unityCatalog.{UnityCatalog, UnityCatalogTableOperations} import shadedForDelta.org.apache.iceberg.util.LocationUtil import org.apache.spark.internal.MDC @@ -54,6 +56,9 @@ sealed trait IcebergConversionMode { // Used by Post-commit Delta UniForm (Iceberg conversion in Delta post commit hook) case object UNIFORM_POST_COMMIT_MODE extends IcebergConversionMode { } +// Used by atomic Delta UniForm +case object UNIFORM_CC_MODE extends IcebergConversionMode { +} /** * Used to prepare (convert) and then commit a set of Delta actions into the Iceberg table located * at the same path as [[postCommitSnapshot]] @@ -73,6 +78,7 @@ class IcebergConversionTransaction( protected val tableOp: IcebergTableOp = WRITE_TABLE, protected val lastConvertedIcebergSnapshotId: Option[Long] = None, protected val lastConvertedDeltaVersion: Option[Long] = None, + protected val lastConvertedIcebergMetadataPath: Option[String] = None, protected val metadataUpdates: java.util.ArrayList[MetadataUpdate] = new java.util.ArrayList[MetadataUpdate]() ) extends DeltaLogging { @@ -456,30 +462,75 @@ class IcebergConversionTransaction( committed = true } + /** + * Retrieves the converted Iceberg metadata location and its current snapshot. + * This method should only be called after a successful table conversion operation + * + * @return A tuple containing: + * - String: The path where the Iceberg metadata file was written + * - IcebergMetadata: The converted Iceberg metadata + * @throws IllegalStateException if the Iceberg metadata has not been converted + * @throws UnsupportedOperationException if called on non-UnityCatalogTableOperations + */ + def getConvertedIcebergMetadata: (String, TableMetadata) = + txn.asInstanceOf[BaseTransaction].underlyingOps() match { + case ops: UnityCatalogTableOperations => + ops.getLastWrittenTableMetadataWithLocation.toScala match { + case Some((metadataPath, tableMetadata)) => + (metadataPath, tableMetadata) + case _ => throw new IllegalStateException( + "Could not get converted Iceberg metadata: new written metadata not found") + } + case _ => + throw new IllegalStateException( + "Could not get converted Iceberg metadata:" + + " underlying UnityCatalogTableOperations not found" + ) + } + /////////////////////// // Protected Methods // /////////////////////// protected def createIcebergTxn(tableOpOpt: Option[IcebergTableOp] = None): IcebergTransaction = { - val hiveCatalog = IcebergTransactionUtils.createHiveCatalog(conf, metadataUpdates) - val icebergTableId = IcebergTransactionUtils - .convertSparkTableIdentifierToIcebergHive(catalogTable.identifier) + val baseMetadataPath = + (tableOpOpt.getOrElse(tableOp), lastConvertedIcebergMetadataPath) match { + case (CREATE_TABLE, None) => None + case (CREATE_TABLE, Some(_)) => + throw new IllegalStateException( + "Unexpected base metadata path for CREATE_TABLE operation") + case (op, None) => + throw new IllegalStateException(s"Missing base metadata path for $op operation") + case (_, Some(path)) => Some(path) + } + + val ucTable = new UnityCatalog( + metadataUpdates, + baseMetadataPath.toJava + ) + ucTable.initialize(null, new java.util.HashMap[String, String]()) + ucTable.setConf(conf) + + val icebergIdentifier = + IcebergTransactionUtils.convertSparkTableIdentifierToIceberg(catalogTable.identifier) - val tableExists = hiveCatalog.tableExists(icebergTableId) + val tableExists = ucTable.tableExists(icebergIdentifier) def tableBuilder = { - hiveCatalog - .buildTable(icebergTableId, icebergSchema) + val tableLocation = postCommitSnapshot.deltaLog.dataPath.toString + ucTable + .buildTable(icebergIdentifier, icebergSchema) .withPartitionSpec(partitionSpec) .withProperties(convert.properties.asJava) + .withLocation(tableLocation) } - tableOpOpt.getOrElse(tableOp) match { + val txn = tableOpOpt.getOrElse(tableOp) match { case WRITE_TABLE => if (tableExists) { recordFrameProfile("IcebergConversionTransaction", "loadTable") { - hiveCatalog.loadTable(icebergTableId).newTransaction() + ucTable.loadTable(icebergIdentifier).newTransaction() } } else { throw new IllegalStateException(s"Cannot write to table $tablePath. Table doesn't exist.") @@ -501,6 +552,7 @@ class IcebergConversionTransaction( throw new IllegalStateException(s"Cannot replace table $tablePath. Table doesn't exist.") } } + txn } //////////////////// diff --git a/iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergConverter.scala b/iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergConverter.scala index bb76cf81c80..e8263ed8cf3 100644 --- a/iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergConverter.scala +++ b/iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergConverter.scala @@ -180,7 +180,7 @@ class IcebergConverter * Convert Iceberg Metadata for a complete snapshot. Used for conversion * after delta commits and in create table */ - private def convertSnapshotInternal( + protected def convertSnapshotInternal( snapshotToConvert: Snapshot, readSnapshotOpt: Option[Snapshot], lastConvertedInfo: LastConvertedIcebergInfo, @@ -321,7 +321,6 @@ class IcebergConverter icebergTxn.commit() logInfo(s"icebergTxn committed for table ${Option(catalogTable).map(_.identifier)} " + s"with converted delta version ${snapshotToConvert.version}") - validateIcebergCommit(snapshotToConvert, catalogTable) recordDeltaEvent( snapshotToConvert.deltaLog, diff --git a/iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergTransactionUtils.scala b/iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergTransactionUtils.scala index b88c14aa3b0..563ef590753 100644 --- a/iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergTransactionUtils.scala +++ b/iceberg/src/main/scala/org/apache/spark/sql/delta/icebergShaded/IcebergTransactionUtils.scala @@ -240,6 +240,24 @@ object IcebergTransactionUtils catalog } + /** + * Encode Spark table identifier to Iceberg table identifier by putting "database" and "catalog" + * to the "namespace" in Iceberg table identifier + */ + def convertSparkTableIdentifierToIceberg( + identifier: SparkTableIdentifier): IcebergTableIdentifier = { + val namespace = (identifier.database, identifier.catalog) match { + case (Some(database), Some(catalog)) => Namespace.of(database, catalog) + case (Some(database), None) => Namespace.of(database) + case (None, Some(catalog)) => + throw new IllegalArgumentException( + "Spark does not allow the constructors to skip the `database` when `catalog` is used" + ) + case (None, None) => Namespace.empty() + } + IcebergTableIdentifier.of(namespace, identifier.table) + } + /** * Encode Spark table identifier to Iceberg table identifier by putting * only "database" to the "namespace" in Iceberg table identifier. diff --git a/iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClient.scala b/iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClient.scala index f800a2d8956..3df29756b19 100644 --- a/iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClient.scala +++ b/iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClient.scala @@ -27,9 +27,12 @@ import org.apache.hadoop.conf.Configuration import org.apache.http.client.methods.{HttpGet, HttpPost} import org.apache.http.entity.{ContentType, StringEntity} import org.apache.http.util.EntityUtils -import org.apache.http.{HttpHeaders, HttpStatus} -import org.apache.http.impl.client.HttpClientBuilder +import org.apache.http.{HttpHeaders, HttpRequest, HttpRequestInterceptor, HttpResponse, HttpStatus} +import org.apache.http.client.ServiceUnavailableRetryStrategy +import org.apache.http.impl.client.{DefaultHttpRequestRetryHandler, HttpClientBuilder} +import org.apache.http.protocol.HttpContext import org.apache.http.message.BasicHeader +import org.apache.spark.internal.Logging import org.apache.spark.sql.delta.util.JsonUtils import org.apache.spark.sql.sources.Filter import org.apache.spark.sql.types.StructType @@ -63,12 +66,14 @@ private case class CatalogConfigResponse( * @param baseUriRaw Base URI of the Iceberg REST catalog up to /v1, e.g., * "http:///iceberg/v1". Trailing slashes are handled automatically. * @param catalogName Name of the catalog for config endpoint query parameter. - * @param token Authentication token for the catalog server. + * @param tokenSupplier Supplier of auth tokens, called per-request to support OAuth. + * Returns empty string if no auth is needed. */ class IcebergRESTCatalogPlanningClient( baseUriRaw: String, catalogName: String, - token: String) extends ServerSidePlanningClient with AutoCloseable { + tokenSupplier: () => String +) extends ServerSidePlanningClient with Logging { // Normalize baseUri to handle trailing slashes private val baseUri = baseUriRaw.stripSuffix("/") @@ -186,8 +191,8 @@ class IcebergRESTCatalogPlanningClient( * Returns None on any error or if no prefix is defined in the config. */ private def fetchCatalogPrefix(): Option[String] = { + val configUri = s"$baseUri/config?warehouse=$catalogName" try { - val configUri = s"$baseUri/config?warehouse=$catalogName" val httpGet = new HttpGet(configUri) val response = httpClient.execute(httpGet) try { @@ -203,23 +208,20 @@ class IcebergRESTCatalogPlanningClient( response.close() } } catch { - case _: Exception => None + case e: Exception => + logWarning(s"Failed to fetch catalog prefix from $configUri. " + + s"Falling back to base URI. Error: ${e.getMessage}") + None } } + // Default headers without auth -- auth is injected per-request via HttpRequestInterceptor private val httpHeaders = { - val baseHeaders = Map( + Map( HttpHeaders.ACCEPT -> ContentType.APPLICATION_JSON.getMimeType, HttpHeaders.CONTENT_TYPE -> ContentType.APPLICATION_JSON.getMimeType, HttpHeaders.USER_AGENT -> buildUserAgent() - ) - // Add Bearer token authentication if token is provided - val headersWithAuth = if (token.nonEmpty) { - baseHeaders + (HttpHeaders.AUTHORIZATION -> s"Bearer $token") - } else { - baseHeaders - } - headersWithAuth.map { case (k, v) => new BasicHeader(k, v) }.toSeq.asJava + ).map { case (k, v) => new BasicHeader(k, v) }.toSeq.asJava } /** @@ -309,10 +311,31 @@ class IcebergRESTCatalogPlanningClient( scala.util.Properties.versionNumberString } - private lazy val httpClient = HttpClientBuilder.create() - .setDefaultHeaders(httpHeaders) - .setConnectionTimeToLive(30, java.util.concurrent.TimeUnit.SECONDS) - .build() + // Maximum number of retries for transient HTTP failures (IOException, 5xx server errors) + private val HTTP_MAX_RETRIES = 3 + + private lazy val httpClient = { + val builder = HttpClientBuilder.create() + .setDefaultHeaders(httpHeaders) + .setConnectionTimeToLive(30, java.util.concurrent.TimeUnit.SECONDS) + // requestSentRetryEnabled=true: safe to retry already-sent requests because + // planScan is a read-only operation (idempotent POST to /plan endpoint) + .setRetryHandler(new DefaultHttpRequestRetryHandler(HTTP_MAX_RETRIES, true)) + .setServiceUnavailableRetryStrategy(new ServerErrorRetryStrategy(HTTP_MAX_RETRIES)) + + // Per-request interceptor: calls tokenSupplier() to get the current token. + // The token provider implementation handles caching as needed. + builder.addInterceptorFirst(new HttpRequestInterceptor { + override def process(request: HttpRequest, context: HttpContext): Unit = { + val token = tokenSupplier() + if (token != null && token.nonEmpty) { + request.setHeader(HttpHeaders.AUTHORIZATION, s"Bearer $token") + } + } + }) + + builder.build() + } override def canConvertFilters(filters: Array[Filter]): Boolean = { // Check if all filters can be converted to Iceberg expressions @@ -369,7 +392,6 @@ class IcebergRESTCatalogPlanningClient( } val httpPost = new HttpPost(planTableScanUri) httpPost.setEntity(new StringEntity(requestJson, ContentType.APPLICATION_JSON)) - // TODO: Add retry logic for transient HTTP failures (e.g., connection timeouts, 5xx errors) val httpResponse = httpClient.execute(httpPost) // Only unpartitioned tables are supported. This map is used when parsing the response @@ -503,6 +525,39 @@ class IcebergRESTCatalogPlanningClient( } } + /** + * Retry strategy for server errors (5xx status codes) with exponential backoff. + * Retries up to maxRetries times with doubling intervals (1s, 2s, 4s, ...). + * Does NOT retry on client errors (4xx) since those indicate request-level issues. + * + * The ServiceUnavailableRetryStrategy interface calls retryRequest() first, then + * getRetryInterval(), so we capture the execution count in retryRequest() and + * use it to compute the backoff in getRetryInterval(). + */ + private class ServerErrorRetryStrategy(maxRetries: Int) + extends ServiceUnavailableRetryStrategy { + + // ThreadLocal so concurrent planScan calls each track their own retry attempt. + // The HTTP client is shared and thread-safe (see class doc), so multiple threads + // can be retrying independently through the same strategy instance. + private val lastExecutionCount = new ThreadLocal[Int] { + override def initialValue(): Int = 1 + } + + override def retryRequest( + response: HttpResponse, + executionCount: Int, + context: HttpContext): Boolean = { + lastExecutionCount.set(executionCount) + val statusCode = response.getStatusLine.getStatusCode + statusCode >= 500 && executionCount <= maxRetries + } + + // Exponential backoff: 1s, 2s, 4s, ... + override def getRetryInterval: Long = + java.util.concurrent.TimeUnit.SECONDS.toMillis(1L << (lastExecutionCount.get() - 1)) + } + private def parsePlanTableScanResponse( json: String, specsById: Map[Int, PartitionSpec], diff --git a/iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClientFactory.scala b/iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClientFactory.scala index aaf0eb35b26..f072b04a625 100644 --- a/iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClientFactory.scala +++ b/iceberg/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClientFactory.scala @@ -28,9 +28,11 @@ class IcebergRESTCatalogPlanningClientFactory extends ServerSidePlanningClientFa metadata: ServerSidePlanningMetadata): ServerSidePlanningClient = { val baseUri = metadata.planningEndpointUri - val token = metadata.authToken.getOrElse("") val catalogName = metadata.catalogName + val supplier: () => String = + metadata.tokenSupplier.getOrElse(() => "") - new IcebergRESTCatalogPlanningClient(baseUri, catalogName, token) + new IcebergRESTCatalogPlanningClient( + baseUri, catalogName, supplier) } } diff --git a/iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTCatalogAdapterWithPlanSupport.java b/iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTCatalogAdapterWithPlanSupport.java index a94dca8cc43..04c4e94d674 100644 --- a/iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTCatalogAdapterWithPlanSupport.java +++ b/iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTCatalogAdapterWithPlanSupport.java @@ -20,6 +20,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import shadedForDelta.org.apache.iceberg.BaseFileScanTask; @@ -81,6 +82,14 @@ class IcebergRESTCatalogAdapterWithPlanSupport extends RESTCatalogAdapter { // Volatile is used to guarantee correct cross-thread access (test thread and Jetty server thread). private static volatile String capturedPlanRequestPath = null; + // Failure injection fields for testing HTTP retry logic. + // planRequestFailCount: number of remaining /plan requests to fail before allowing success. + // planRequestFailStatusCode: HTTP status code to return for injected failures. + // planRequestCount: total number of /plan requests received (for verifying retry behavior). + private static final AtomicInteger planRequestFailCount = new AtomicInteger(0); + private static volatile int planRequestFailStatusCode = 503; + private static final AtomicInteger planRequestCount = new AtomicInteger(0); + IcebergRESTCatalogAdapterWithPlanSupport(Catalog catalog) { super(catalog); this.catalog = catalog; @@ -189,6 +198,54 @@ static void clearCaptured() { testCredentials = null; testResidual = null; capturedPlanRequestPath = null; + planRequestFailCount.set(0); + planRequestCount.set(0); + } + + /** + * Configure the server to fail the next N /plan requests with the specified HTTP status code. + * After N failures, subsequent requests proceed normally. + * Used for testing HTTP retry logic in the client. + * + * @param count Number of /plan requests to fail + * @param statusCode HTTP status code to return for injected failures (e.g., 503, 404) + */ + static void setFailNextPlanRequests(int count, int statusCode) { + planRequestFailCount.set(count); + planRequestFailStatusCode = statusCode; + } + + /** + * Atomically get and decrement the remaining failure count. + * Returns the value before decrement. If > 0, the request should be failed. + * Package-private for servlet access. + */ + static int getAndDecrementFailCount() { + return planRequestFailCount.getAndDecrement(); + } + + /** + * Get the HTTP status code to use for injected failures. + * Package-private for servlet access. + */ + static int getPlanRequestFailStatusCode() { + return planRequestFailStatusCode; + } + + /** + * Increment and return the total /plan request count. + * Package-private for servlet access. + */ + static void incrementPlanRequestCount() { + planRequestCount.incrementAndGet(); + } + + /** + * Get the total number of /plan requests received. + * Package-private for test access. + */ + static int getPlanRequestCount() { + return planRequestCount.get(); } @Override diff --git a/iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTServer.java b/iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTServer.java index 5bbe5b2b95c..378475676b0 100644 --- a/iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTServer.java +++ b/iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTServer.java @@ -215,6 +215,26 @@ public void setTestResidual(shadedForDelta.org.apache.iceberg.expressions.Expres IcebergRESTCatalogAdapterWithPlanSupport.setTestResidual(residual); } + /** + * Configure the server to fail the next N /plan requests with the specified HTTP status code. + * After N failures, subsequent requests proceed normally. + * Used for testing HTTP retry logic in the client. + * + * @param count Number of /plan requests to fail + * @param statusCode HTTP status code to return for injected failures (e.g., 503, 404) + */ + public void setFailNextPlanRequests(int count, int statusCode) { + IcebergRESTCatalogAdapterWithPlanSupport.setFailNextPlanRequests(count, statusCode); + } + + /** + * Get the total number of /plan requests received since last clearCaptured(). + * Used for verifying retry behavior in tests. + */ + public int getPlanRequestCount() { + return IcebergRESTCatalogAdapterWithPlanSupport.getPlanRequestCount(); + } + /** * Clear captured filter and projection. Call between tests. */ diff --git a/iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTServletWithPlanSupport.java b/iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTServletWithPlanSupport.java index 6f59cc35895..6617850401f 100644 --- a/iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTServletWithPlanSupport.java +++ b/iceberg/src/test/java/shadedForDelta/org/apache/iceberg/rest/IcebergRESTServletWithPlanSupport.java @@ -121,6 +121,23 @@ private void handleConfigRequest(HttpServletRequest req, HttpServletResponse res private void handlePlanRequest(HttpServletRequest req, HttpServletResponse resp) throws IOException { + // Track plan request count for testing retry behavior + IcebergRESTCatalogAdapterWithPlanSupport.incrementPlanRequestCount(); + + // Check if we should inject a failure for testing HTTP retry logic + int remainingFailures = IcebergRESTCatalogAdapterWithPlanSupport.getAndDecrementFailCount(); + if (remainingFailures > 0) { + int failStatusCode = IcebergRESTCatalogAdapterWithPlanSupport.getPlanRequestFailStatusCode(); + LOG.info("Injecting test failure: returning HTTP {} ({} failures remaining)", + failStatusCode, remainingFailures - 1); + resp.setStatus(failStatusCode); + resp.setContentType("application/json"); + resp.getWriter().write( + "{\"error\": {\"message\": \"Injected test failure\", \"type\": \"TestError\", \"code\": " + + failStatusCode + "}}"); + return; + } + try { // Extract request components String path = req.getPathInfo(); diff --git a/iceberg/src/test/scala/org/apache/spark/sql/delta/ConvertIcebergToDeltaSuite.scala b/iceberg/src/test/scala/org/apache/spark/sql/delta/ConvertIcebergToDeltaSuite.scala index b337076f91e..4338a819637 100644 --- a/iceberg/src/test/scala/org/apache/spark/sql/delta/ConvertIcebergToDeltaSuite.scala +++ b/iceberg/src/test/scala/org/apache/spark/sql/delta/ConvertIcebergToDeltaSuite.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.delta // scalastyle:off import.ordering.noEmptyLine -import java.io.File +import java.io.{ByteArrayOutputStream, File} import java.text.SimpleDateFormat import java.util.TimeZone @@ -32,6 +32,8 @@ import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.StatsUtils import io.delta.sql.DeltaSparkSessionExtension import org.apache.hadoop.fs.Path +import org.apache.avro.file.{DataFileReader, DataFileWriter, SeekableByteArrayInput} +import org.apache.avro.generic.{GenericDatumReader, GenericDatumWriter, GenericRecord} import org.apache.iceberg.{Table, TableProperties} import org.apache.iceberg.hadoop.HadoopTables import org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions @@ -591,6 +593,83 @@ trait ConvertIcebergToDeltaSuiteBase } } + /** + * Strips the "schema" metadata key from all manifest Avro files of the table's current snapshot. + * This simulates a V2 writer that omits the optional schema field. Without passing specsById + * to ManifestFiles.read, reading such manifests causes an NPE in SchemaParser.fromJson. + */ + private def stripSchemaFromManifests(table: Table): Unit = { + val manifests = table.currentSnapshot().dataManifests(table.io()).asScala + // scalastyle:off deltahadoopconfiguration + val conf = spark.sessionState.newHadoopConf() + // scalastyle:on deltahadoopconfiguration + manifests.foreach { manifest => + val path = new Path(manifest.path()) + val fs = path.getFileSystem(conf) + + // Read the entire manifest file into a byte array + val inputStream = fs.open(path) + val bytes = try { + org.apache.commons.io.IOUtils.toByteArray(inputStream) + } finally { + inputStream.close() + } + + val datumReader = new GenericDatumReader[GenericRecord]() + val reader = new DataFileReader[GenericRecord]( + new SeekableByteArrayInput(bytes), datumReader) + + // Collect records and metadata + val records = new java.util.ArrayList[GenericRecord]() + while (reader.hasNext) { + records.add(reader.next()) + } + val avroSchema = reader.getSchema + val metaKeys = reader.getMetaKeys.asScala.toSeq + + // Write back without the "schema" metadata key + val out = new ByteArrayOutputStream() + val datumWriter = new GenericDatumWriter[GenericRecord](avroSchema) + val writer = new DataFileWriter[GenericRecord](datumWriter) + val reservedKeys = Set("schema", "avro.schema", "avro.codec") + metaKeys.filterNot(reservedKeys.contains).foreach { key => + writer.setMeta(key, reader.getMeta(key)) + } + writer.create(avroSchema, out) + records.asScala.foreach(writer.append) + writer.close() + reader.close() + + // Overwrite the original file + val outputStream = fs.create(path, true) + try { + outputStream.write(out.toByteArray) + } finally { + outputStream.close() + } + } + } + + test("convert Iceberg table with manifest missing schema metadata") { + withTable(table) { + spark.sql( + s"""CREATE TABLE $table (id bigint, data string) + |USING iceberg PARTITIONED BY (data) + |TBLPROPERTIES ("format-version" = "2")""".stripMargin) + Seq((1L, "a"), (2L, "b")).toDF("id", "data") + .write.format("iceberg").mode("append").saveAsTable(table) + // Strip the "schema" metadata from manifest Avro files to simulate a V2 writer + // that omits the optional schema field. Without passing specsById to + // ManifestFiles.read, this causes an NPE in SchemaParser.fromJson. + val iceTable = readIcebergHadoopTable(tablePath) + stripSchemaFromManifests(iceTable) + convert(s"iceberg.`$tablePath`") + checkAnswer( + spark.read.format("delta").load(tablePath), + Row(1, "a") :: Row(2, "b") :: Nil) + } + } + test("convert Iceberg table with not null columns") { withTable(table) { spark.sql( diff --git a/iceberg/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClientSuite.scala b/iceberg/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClientSuite.scala index 2243abb843c..af96b017ebb 100644 --- a/iceberg/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClientSuite.scala +++ b/iceberg/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/IcebergRESTCatalogPlanningClientSuite.scala @@ -78,7 +78,7 @@ class IcebergRESTCatalogPlanningClientSuite extends QueryTest with SharedSparkSe // Tests that the REST /plan endpoint returns 0 files for an empty table. test("basic plan table scan via IcebergRESTCatalogPlanningClient") { withTempTable("testTable") { table => - val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", "") + val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", () => "") try { val scanPlan = client.planScan(defaultNamespace.toString, "testTable") assert(scanPlan != null, "Scan plan should not be null") @@ -106,7 +106,7 @@ class IcebergRESTCatalogPlanningClientSuite extends QueryTest with SharedSparkSe .map(row => (new Path(row.getString(0)).getName, row.getLong(1))) .toMap - val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", "") + val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", () => "") try { val scanPlan = client.planScan(defaultNamespace.toString, "tableWithData") assert(scanPlan != null, "Scan plan should not be null") @@ -145,7 +145,7 @@ class IcebergRESTCatalogPlanningClientSuite extends QueryTest with SharedSparkSe withTempTable("testTable") { table => // Client expects baseUri to include the /v1 path (per Iceberg REST spec) - val client = new IcebergRESTCatalogPlanningClient(s"$serverUri/v1", "test_catalog", "") + val client = new IcebergRESTCatalogPlanningClient(s"$serverUri/v1", "test_catalog", () => "") try { // Make a call that will trigger the lazy initialization of icebergRestCatalogUriRoot // which internally calls fetchCatalogPrefix() @@ -173,7 +173,7 @@ class IcebergRESTCatalogPlanningClientSuite extends QueryTest with SharedSparkSe withTempTable("testTable") { table => // Client expects baseUri to include the /v1 path (per Iceberg REST spec) - val client = new IcebergRESTCatalogPlanningClient(s"$serverUri/v1", "test_catalog", "") + val client = new IcebergRESTCatalogPlanningClient(s"$serverUri/v1", "test_catalog", () => "") try { // Make a call that will trigger the lazy initialization val scanPlan = client.planScan(defaultNamespace.toString, "testTable") @@ -200,7 +200,7 @@ class IcebergRESTCatalogPlanningClientSuite extends QueryTest with SharedSparkSe withTempTable("filterTest") { table => populateTestData(s"rest_catalog.${defaultNamespace}.filterTest") - val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", "") + val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", () => "") try { val testCases = Seq( (EqualTo("longCol", 2L), "EqualTo numeric (long)"), @@ -310,7 +310,7 @@ class IcebergRESTCatalogPlanningClientSuite extends QueryTest with SharedSparkSe Set("`address.city`")) ) - val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", "") + val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", () => "") try { testCases.foreach { testCase => // Clear previous captured projection @@ -343,7 +343,7 @@ class IcebergRESTCatalogPlanningClientSuite extends QueryTest with SharedSparkSe val tableName = s"rest_catalog.${defaultNamespace}.limitTest" populateTestData(tableName) - val client = new IcebergRESTCatalogPlanningClient(serverUri, null, "") + val client = new IcebergRESTCatalogPlanningClient(serverUri, null, () => "") try { // Test different limit values val testCases = Seq( @@ -378,7 +378,7 @@ class IcebergRESTCatalogPlanningClientSuite extends QueryTest with SharedSparkSe val tableName = s"rest_catalog.${defaultNamespace}.filterProjectionLimitTest" populateTestData(tableName) - val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", "") + val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", () => "") try { // Note: Filter types are already tested in "filter sent to IRC server" test. // Here we verify filter, projection, AND limit are sent together correctly. @@ -456,7 +456,7 @@ class IcebergRESTCatalogPlanningClientSuite extends QueryTest with SharedSparkSe withTempTable("caseSensitiveTest") { table => populateTestData(s"rest_catalog.${defaultNamespace}.caseSensitiveTest") - val client = new IcebergRESTCatalogPlanningClient(serverUri, null, "") + val client = new IcebergRESTCatalogPlanningClient(serverUri, null, () => "") try { server.clearCaptured() @@ -481,7 +481,7 @@ class IcebergRESTCatalogPlanningClientSuite extends QueryTest with SharedSparkSe withTempTable("residualTest") { table => populateTestData(s"rest_catalog.${defaultNamespace}.residualTest") - val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", "") + val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", () => "") try { // Verify that a trivial (alwaysTrue) residual is accepted server.setTestResidual(Expressions.alwaysTrue()) @@ -523,8 +523,110 @@ class IcebergRESTCatalogPlanningClientSuite extends QueryTest with SharedSparkSe IcebergRESTServerTestUtils.populateTestData(spark, tableName) } + test("retry on transient 503 server error") { + withTempTable("retryTest503") { table => + populateTestData(s"rest_catalog.${defaultNamespace}.retryTest503") + + val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", () => "") + try { + server.clearCaptured() + // Configure server to fail the first plan request with 503 + server.setFailNextPlanRequests(1, 503) + + // Client should retry and succeed on the second attempt + val scanPlan = client.planScan(defaultNamespace.toString, "retryTest503") + assert(scanPlan != null, "Scan plan should not be null after retry") + assert(scanPlan.files.nonEmpty, "Scan plan should have files after successful retry") + + // Verify 2 requests were made: 1 failed (503) + 1 success + assert(server.getPlanRequestCount() == 2, + s"Expected 2 plan requests (1 retry), got ${server.getPlanRequestCount()}") + } finally { + server.clearCaptured() + client.close() + } + } + } + + test("retries exhausted on persistent 503 server error") { + // No populateTestData needed: failure injection intercepts at the servlet level before + // table data is accessed, so we only need the table to exist for a valid URI. + withTempTable("retryTestExhausted") { table => + val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", () => "") + try { + server.clearCaptured() + // Configure server to fail more requests than the client will retry (max 3 retries = 4 + // total attempts). Setting 10 failures ensures all retries see 503. + server.setFailNextPlanRequests(10, 503) + + val exception = intercept[java.io.IOException] { + client.planScan(defaultNamespace.toString, "retryTestExhausted") + } + assert(exception.getMessage.contains("503"), + s"Error should mention 503 status code. Got: ${exception.getMessage}") + + // Verify 4 requests were made: 1 original + 3 retries (max retries = 3) + assert(server.getPlanRequestCount() == 4, + s"Expected 4 plan requests (1 + 3 retries), got ${server.getPlanRequestCount()}") + } finally { + server.clearCaptured() + client.close() + } + } + } + + test("no retry on 404 client error") { + // No populateTestData needed: failure injection intercepts at the servlet level before + // table data is accessed, so we only need the table to exist for a valid URI. + withTempTable("retryTest404") { table => + val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", () => "") + try { + server.clearCaptured() + // Configure server to fail all plan requests with 404 + // Using a high count ensures the test fails if the client retries + server.setFailNextPlanRequests(10, 404) + + // Client should NOT retry 404 and should throw immediately + val exception = intercept[java.io.IOException] { + client.planScan(defaultNamespace.toString, "retryTest404") + } + assert(exception.getMessage.contains("404"), + s"Error should mention 404 status code. Got: ${exception.getMessage}") + + // Verify only 1 request was made (no retry for 404) + assert(server.getPlanRequestCount() == 1, + s"Expected 1 plan request (no retry for 404), got ${server.getPlanRequestCount()}") + } finally { + server.clearCaptured() + client.close() + } + } + } + + test("fetchCatalogPrefix falls back to baseUri on connection failure") { + // Use a port that's expected to have no listener. fetchCatalogPrefix() makes an HTTP GET + // to /config which will fail with a connection error. It should catch the exception, log a + // warning, and return None — causing icebergRestCatalogUriRoot to fall back to baseUri. + // The subsequent planScan HTTP POST will also fail (same unreachable host). + val unreachableUri = "http://localhost:1" + val client = new IcebergRESTCatalogPlanningClient(unreachableUri, "test_catalog", () => "") + try { + val ex = intercept[Exception] { + client.planScan("test_db", "test_table") + } + // Verify the exception is a connection error. This confirms fetchCatalogPrefix() + // did not throw a different exception type (e.g., NPE, parse error) and that the + // client progressed past the config fetch to attempt the plan HTTP POST. + assert(ex.getMessage != null, + "Expected a connection error with a message from the HTTP client") + } finally { + client.close() + } + } + test("User-Agent header format") { - val client = new IcebergRESTCatalogPlanningClient("http://localhost:8080", "test_catalog", "") + val client = new IcebergRESTCatalogPlanningClient( + "http://localhost:8080", "test_catalog", () => "") try { val userAgent = client.getUserAgent() @@ -563,4 +665,61 @@ class IcebergRESTCatalogPlanningClientSuite extends QueryTest with SharedSparkSe } } + test("e2e: OAuth token supplier with mock token server") { + // Start a mock OAuth token endpoint using JDK HttpServer. + // OAuthTokenProvider sends: POST with Basic auth + grant_type=client_credentials + val tokenRequestCount = new java.util.concurrent.atomic.AtomicInteger(0) + val mockToken = "mock-oauth-access-token" + val oauthServer = com.sun.net.httpserver.HttpServer.create( + new java.net.InetSocketAddress("localhost", 0), 0) + oauthServer.createContext("/token", (exchange: com.sun.net.httpserver.HttpExchange) => { + try { + tokenRequestCount.incrementAndGet() + val responseJson = + s"""{"access_token":"$mockToken","token_type":"Bearer","expires_in":3600}""" + val bytes = responseJson.getBytes(java.nio.charset.StandardCharsets.UTF_8) + exchange.getResponseHeaders.add("Content-Type", "application/json") + exchange.sendResponseHeaders(200, bytes.length) + exchange.getResponseBody.write(bytes) + } finally { + exchange.close() + } + }) + oauthServer.start() + + try { + val oauthUri = s"http://localhost:${oauthServer.getAddress.getPort}/token" + val authConfig = Map( + "type" -> "oauth", + "oauth.uri" -> oauthUri, + "oauth.clientId" -> "test-client-id", + "oauth.clientSecret" -> "test-client-secret") + val tokenProvider = io.unitycatalog.client.auth.TokenProvider.create( + authConfig.asJava.asInstanceOf[java.util.Map[String, String]]) + val supplier: () => String = () => tokenProvider.accessToken() + + withTempTable("oauthE2eTest") { table => + val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", supplier) + try { + // First planScan triggers token fetch from mock server + val scanPlan = client.planScan(defaultNamespace.toString, "oauthE2eTest") + assert(scanPlan != null) + + // OAuthTokenProvider should have fetched exactly 1 token (cached for reuse) + assert(tokenRequestCount.get() == 1, + s"Expected 1 token request, got ${tokenRequestCount.get()}") + + // Second planScan reuses cached token (not expired) + client.planScan(defaultNamespace.toString, "oauthE2eTest") + assert(tokenRequestCount.get() == 1, + s"Token should be cached. Expected 1 request, got ${tokenRequestCount.get()}") + } finally { + client.close() + } + } + } finally { + oauthServer.stop(0) + } + } + } diff --git a/iceberg/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlanningCredentialsSuite.scala b/iceberg/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlanningCredentialsSuite.scala index a8808aa06ac..e02cd9b4a9a 100644 --- a/iceberg/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlanningCredentialsSuite.scala +++ b/iceberg/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlanningCredentialsSuite.scala @@ -70,7 +70,7 @@ class ServerSidePlanningCredentialsSuite extends QueryTest with SharedSparkSessi withTempTable("credentialsTest") { table => populateTestData(s"rest_catalog.${defaultNamespace}.credentialsTest") - val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", "") + val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", () => "") try { // Covers the successful credential extraction and Hadoop configuration injection cases. val testCases: Seq[CredentialTestCase] = Seq( @@ -147,7 +147,7 @@ class ServerSidePlanningCredentialsSuite extends QueryTest with SharedSparkSessi withTempTable("incompleteCredsTest") { table => populateTestData(s"rest_catalog.${defaultNamespace}.incompleteCredsTest") - val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", "") + val client = new IcebergRESTCatalogPlanningClient(serverUri, "test_catalog", () => "") try { // Test cases for incomplete credentials that should throw errors val errorTestCases = Seq( diff --git a/iceberg/src/test/scala/org/apache/spark/sql/delta/uniform/UniFormConverterSuite.scala b/iceberg/src/test/scala/org/apache/spark/sql/delta/uniform/UniFormConverterSuite.scala new file mode 100644 index 00000000000..707be81a488 --- /dev/null +++ b/iceberg/src/test/scala/org/apache/spark/sql/delta/uniform/UniFormConverterSuite.scala @@ -0,0 +1,117 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.uniform + +import shadedForDelta.org.apache.iceberg.hadoop.HadoopTables + +import org.apache.spark.sql.QueryTest +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.catalog.CatalogTable +import org.apache.spark.sql.delta.{DeltaLog, Snapshot} +import org.apache.spark.sql.delta.icebergShaded.{IcebergConverter, UNIFORM_CC_MODE} +import org.apache.spark.sql.delta.test.DeltaSQLCommandTest +import org.apache.spark.sql.test.SharedSparkSession + +class IcebergConverterForTest extends IcebergConverter { + def convertSnapshotAndReturnMetadataPath( + snapshotToConvert: Snapshot, + catalogTable: CatalogTable): String = { + val icebergTxn = convertSnapshotInternal( + snapshotToConvert, + readSnapshotOpt = None, + lastConvertedInfo = LastConvertedIcebergInfo( + icebergTable = None, + icebergSnapshotId = None, + deltaVersionConverted = None, + baseMetadataLocationOpt = None + ), + conversionContext = new ConversionContext( + conversionMode = UNIFORM_CC_MODE, + additionalDeltaActionsToCommit = None, + opType = "delta.iceberg.conversion.convertSnapshot" + ), + catalogTable + ) + icebergTxn.getConvertedIcebergMetadata._1 + } +} + +class UniFormConverterSuite extends QueryTest with SharedSparkSession with DeltaSQLCommandTest { + test("convertSnapshot writes Iceberg metadata and file count matches Delta snapshot") { + val tableName = "test_iceberg_converter" + withTable(tableName) { + spark.sql( + s"""CREATE TABLE $tableName (id INT, name STRING) USING DELTA + |TBLPROPERTIES ( + | 'delta.columnMapping.mode' = 'name', + | 'delta.enableIcebergCompatV2' = 'true', + | 'delta.universalFormat.enabledFormats' = 'iceberg' + |)""".stripMargin) + // Do write + spark.sql(s"INSERT INTO $tableName VALUES (1, 'alice'), (2, 'bob'), (3, 'carol')") + val tableId = TableIdentifier(tableName) + val deltaLog = DeltaLog.forTable(spark, tableId) + val snapshot = deltaLog.update() + val catalogTable = spark.sessionState.catalog.getTableMetadata(tableId) + // Trigger conversion + val converter = new IcebergConverterForTest() + val metadataPath = converter.convertSnapshotAndReturnMetadataPath(snapshot, catalogTable) + // Check match + val icebergTable = + new HadoopTables(deltaLog.newDeltaHadoopConf()).load(metadataPath) + val numFilesInIceberg = + icebergTable.currentSnapshot().summary().get("total-data-files").toInt + assert( + numFilesInIceberg == snapshot.numOfFiles, + s"Iceberg total-data-files ($numFilesInIceberg) must equal " + + s"Delta numOfFiles (${snapshot.numOfFiles})") + } + } + + test("convertSnapshot file count matches Delta snapshot after multiple inserts") { + val tableName = "test_iceberg_converter_multi" + withTable(tableName) { + spark.sql( + s"""CREATE TABLE $tableName (id INT) USING DELTA + |TBLPROPERTIES ( + | 'delta.columnMapping.mode' = 'name', + | 'delta.enableIcebergCompatV2' = 'true', + | 'delta.universalFormat.enabledFormats' = 'iceberg' + |)""".stripMargin) + // Do some writes + spark.sql(s"INSERT INTO $tableName VALUES (1)") + spark.sql(s"INSERT INTO $tableName VALUES (2)") + spark.sql(s"INSERT INTO $tableName VALUES (3)") + val tableId = TableIdentifier(tableName) + val deltaLog = DeltaLog.forTable(spark, tableId) + val snapshot = deltaLog.update() + val catalogTable = spark.sessionState.catalog.getTableMetadata(tableId) + assert(snapshot.numOfFiles == 3) + // Trigger conversion + val converter = new IcebergConverterForTest() + val metadataPath = converter.convertSnapshotAndReturnMetadataPath(snapshot, catalogTable) + // Check match + val icebergTable = new HadoopTables(deltaLog.newDeltaHadoopConf()).load(metadataPath) + val numFilesInIceberg = + icebergTable.currentSnapshot().summary().get("total-data-files").toInt + assert( + numFilesInIceberg == snapshot.numOfFiles, + s"Iceberg total-data-files ($numFilesInIceberg) must equal " + + s"Delta numOfFiles (${snapshot.numOfFiles})") + } + } +} diff --git a/icebergShaded/src/main/java/org/apache/iceberg/unityCatalog/UnityCatalog.java b/icebergShaded/src/main/java/org/apache/iceberg/unityCatalog/UnityCatalog.java new file mode 100644 index 00000000000..39785230741 --- /dev/null +++ b/icebergShaded/src/main/java/org/apache/iceberg/unityCatalog/UnityCatalog.java @@ -0,0 +1,140 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iceberg.unityCatalog; + +import java.io.Closeable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.apache.iceberg.BaseMetastoreCatalog; +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.MetadataUpdate; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.hadoop.Configurable; +import org.apache.iceberg.io.CloseableGroup; +import org.apache.iceberg.io.FileIO; + +import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.exceptions.NotFoundException; + + +/** + * UnityCatalog manages delta table with iceberg metadata conversion in unity catalog + * Only newTableOps needs implementation as it is used to commit to Iceberg. All other methods are + * not required (eg, listTables, dropTable, renameTable) because the tables are managed by + * unity catalog outside of iceberg context. + */ +public class UnityCatalog extends BaseMetastoreCatalog implements Closeable, Configurable { + private static final String DEFAULT_FILE_IO_IMPL = "org.apache.iceberg.hadoop.HadoopFileIO"; + + + // Injectable factory for testing purposes. + static class FileIOFactory { + public FileIO newFileIO(String impl, Map properties, Object hadoopConf) { + return CatalogUtil.loadFileIO(impl, properties, hadoopConf); + } + } + + private Object conf; + + private Map catalogProperties; + + private FileIOFactory fileIOFactory; + + private CloseableGroup closeableGroup; + + private List metadataUpdates; + + // If set, all table option in this catalog will be built based on the snapshot baseMetadataLocation points to. + private final Optional baseMetadataLocation; + + public UnityCatalog( + List metadataUpdates + , Optional baseMetadataLocation + ) { + this.metadataUpdates = metadataUpdates; + this.baseMetadataLocation = baseMetadataLocation; + } + + @Override + protected TableOperations newTableOps(TableIdentifier tableIdentifier) { + FileIO fileIO = fileIOFactory.newFileIO(DEFAULT_FILE_IO_IMPL, catalogProperties, conf); + closeableGroup.addCloseable(fileIO); + return new UnityCatalogTableOperations( + fileIO + , tableIdentifier + , metadataUpdates + , baseMetadataLocation + ); + } + + @Override + public void initialize(String name, Map properties) { + this.catalogProperties = properties; + this.fileIOFactory = new FileIOFactory(); + this.closeableGroup = new CloseableGroup(); + } + + @Override + public void close() throws IOException { + if (closeableGroup != null) { + closeableGroup.close(); + } + } + + @Override + protected String defaultWarehouseLocation(TableIdentifier tableIdentifier) { + throw new UnsupportedOperationException( + "UnityCatalog does not currently support defaultWarehouseLocation"); + } + + @Override + public void setConf(Object conf) { + this.conf = conf; + } + + @Override + public List listTables(Namespace namespace) { + throw new UnsupportedOperationException("UnityCatalog does not currently support listTables"); + } + + @Override + public boolean dropTable(TableIdentifier identifier, boolean purge) { + throw new UnsupportedOperationException("UnityCatalog does not currently support dropTable"); + } + + @Override + public void renameTable(TableIdentifier from, TableIdentifier to) { + throw new UnsupportedOperationException("UnityCatalog does not currently support renameTable"); + } + + // If the given metadataLocation is invalid, we also see it as the table doesn't exist + @Override + public boolean tableExists(TableIdentifier identifier) { + try { + loadTable(identifier); + return true; + } catch (NoSuchTableException | NotFoundException e) { + return false; + } + } +} diff --git a/icebergShaded/src/main/java/org/apache/iceberg/unityCatalog/UnityCatalogTableOperations.java b/icebergShaded/src/main/java/org/apache/iceberg/unityCatalog/UnityCatalogTableOperations.java new file mode 100644 index 00000000000..4bc3d534636 --- /dev/null +++ b/icebergShaded/src/main/java/org/apache/iceberg/unityCatalog/UnityCatalogTableOperations.java @@ -0,0 +1,203 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iceberg.unityCatalog; + +import java.net.URI; +import java.net.URISyntaxException; +import java.time.Instant; +import java.util.*; + +import org.apache.iceberg.BaseMetastoreTableOperations; +import org.apache.iceberg.MetadataUpdate; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.CommitFailedException; +import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import scala.Option; +import scala.Tuple2; + +/** + * UnityCatalogTableOperations supports load and commit to Iceberg metadata through unity catalog + */ +public class UnityCatalogTableOperations extends BaseMetastoreTableOperations { + + private static final Logger LOG = LoggerFactory.getLogger(UnityCatalogTableOperations.class); + + /** + * Property to be set in translated Iceberg metadata files. + * Indicates the delta commit version # that it corresponds to. + */ + public static final String DELTA_VERSION_PROPERTY = "delta-version"; + public static final String DELTA_TIMESTAMP_PROPERTY = "delta-timestamp"; + public static final String BASE_DELTA_VERSION_PROPERTY = "base-delta-version"; + public static final String DELTA_HIGH_WATER_MARK_PROPERTY = "delta-high-water-mark"; + + public static final String DELTA_SQL_CONF_BYPASS_SEQUENCE_NUMBER_CHECK = + "spark.databricks.delta.uniform.bypassSnapshotSequenceNumberCheck"; + public static final String DELTA_SQL_CONF_BYPASS_FORMAT_VERSION_DOWNGRAGDE_CHECK = + "spark.databricks.delta.uniform.bypassFormatVersionDowngradeCheck"; + + private final FileIO fileIO; + private final TableIdentifier tableIdentifier; + private final Optional baseMetadataLocation; + private List metadataUpdates; + private Optional> lastWrittenTableMetadataWithLocation; + + public UnityCatalogTableOperations( + FileIO fileIO + , TableIdentifier tableIdentifier + , List metadataUpdates + , Optional baseMetadataLocation + ) { + this.fileIO = fileIO; + this.tableIdentifier = tableIdentifier; + this.metadataUpdates = metadataUpdates; + this.baseMetadataLocation = baseMetadataLocation; + this.lastWrittenTableMetadataWithLocation = Optional.empty(); + } + + @Override + public FileIO io() { + return fileIO; + } + + @Override + protected String tableName() { + return tableIdentifier.toString(); + } + + @Override + public void doCommit(TableMetadata base, TableMetadata metadata) { + TableMetadata.Builder builder = TableMetadata.buildFrom(metadata); + + // hasAddPartitionSpec indicates if the current commit attempt have added a new + // partition spec through metadata update. + boolean hasAddPartitionSpec = false; + boolean initialPartitionSpecExistsAndIsPartitioned = false; + for (PartitionSpec spec : metadata.specs()) { + if (spec.specId() == 0 && spec.isPartitioned()) { + initialPartitionSpecExistsAndIsPartitioned = true; + } + } + Long deltaVersion = Long.parseLong( + metadata.properties().get(DELTA_VERSION_PROPERTY)); + String baseDeltaVersionStr = + metadata.properties().get(BASE_DELTA_VERSION_PROPERTY); + + String deltaHighWaterMarkStr = + metadata.properties().get(DELTA_HIGH_WATER_MARK_PROPERTY); + + Schema lastAddedSchema = metadata.schema(); + for (MetadataUpdate update : metadataUpdates) { + // iceberg-core reassigns field id in its schema when firstly creates table metadata; + // we should always use the schema (with field ids assigned by delta) from MetadataUpdate + // because the parquet data files are already written with field ids assigned by Delta. + if (update instanceof MetadataUpdate.AddSchema) { + MetadataUpdate.AddSchema addSchema = (MetadataUpdate.AddSchema) update; + // lastColumnId must be monotonically increasing. + builder.setCurrentSchema( + addSchema.schema(), Math.max(metadata.lastColumnId(), addSchema.lastColumnId())); + lastAddedSchema = addSchema.schema(); + } else if (update instanceof MetadataUpdate.AddPartitionSpec) { + // Use the partition spec from MetadataUpdate because the partition spec contains + // the correct field ids assigned by Delta + PartitionSpec specToAdd = ((MetadataUpdate.AddPartitionSpec) update).spec().bind(lastAddedSchema); + if (!specToAdd.compatibleWith(metadata.spec())) { + builder.setDefaultPartitionSpec(specToAdd); + hasAddPartitionSpec = true; + } + } else { + update.applyTo(builder); + } + } + + // Remove the initial partitioned partition spec (id=0) if new partition spec is added + // because the initial partition spec has field ids assigned by iceberg-core + // which may mismatch with field id assigned by Delta + if (hasAddPartitionSpec && initialPartitionSpecExistsAndIsPartitioned) { + MetadataUpdate.RemovePartitionSpecs removeSpecs = new MetadataUpdate.RemovePartitionSpecs(Set.of(0)); + removeSpecs.applyTo(builder); + } + + metadata = builder.build(); + + if (base != current()) { + throw new CommitFailedException("Cannot commit changes based on stale table metadata"); + } + if (base == metadata) { + LOG.info("Nothing to commit."); + return; + } + + final int newVersion = currentVersion() + 1; + String newMetadataLocation = writeNewMetadata(metadata, newVersion); + lastWrittenTableMetadataWithLocation = Optional.of(Tuple2.apply(newMetadataLocation, metadata)); + } + + @Override + public TableMetadata refresh() { + // during Delta to iceberg conversion, the table should always exist in catalog and the only + // case with NoSuchTableException is the very first metadata conversion where iceberg + // metadata and metadata location table prop does not exist yet. + try { + return super.refresh(); + } catch (NoSuchTableException e) { + return null; + } + } + + @Override + public void doRefresh() { + LOG.debug("Getting metadata location for table {}", tableIdentifier); + String location = loadTableMetadataLocation(); + Preconditions.checkState( + location != null && !location.isEmpty(), + "Got null or empty location %s for table %s", + location, + tableIdentifier); + refreshFromMetadataLocation(location); + } + + /** + * Returns both the location of the Iceberg metadata file and its corresponding table metadata. + * + * @return An Optional containing a tuple of: + * - String: The path where the metadata file was written + * - TableMetadata: The corresponding Iceberg table metadata + */ + public Optional> getLastWrittenTableMetadataWithLocation() { + return lastWrittenTableMetadataWithLocation; + } + + private String loadTableMetadataLocation() { + if (baseMetadataLocation.isPresent()) { + return baseMetadataLocation.get(); + } + throw new NoSuchTableException("Cannot find iceberg table %s. Either the table does " + + "not exist or the corresponding Delta table has not been converted yet.", + tableIdentifier.toString()); + } + +} diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/expressions/Literal.java b/kernel/kernel-api/src/main/java/io/delta/kernel/expressions/Literal.java index 4f111affdff..5dac2ef78d7 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/expressions/Literal.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/expressions/Literal.java @@ -193,6 +193,25 @@ public static Literal ofDecimal(BigDecimal value, int precision, int scale) { return new Literal(valueToStore, new DecimalType(precision, scale)); } + /** + * Create a geospatial Well-Known Text (WKT) literal with the given data type. This factory only + * enforces the type contract (GeometryType or GeographyType). Callers are responsible for WKT + * content validation, as different engines may support different geometry types beyond POINT. + * + * @param wkt WKT string value + * @param dataType must be GeometryType or GeographyType + * @return a Literal with the given geospatial type and String value + */ + public static Literal ofGeospatialWKT(String wkt, DataType dataType) { + Objects.requireNonNull( + wkt, "WKT string cannot be null; use Literal.ofNull()" + " for null values"); + checkArgument( + dataType instanceof GeometryType || dataType instanceof GeographyType, + "dataType must be GeometryType or GeographyType, got: %s", + dataType); + return new Literal(wkt, dataType); + } + /** * Create {@code null} value literal. * @@ -232,6 +251,8 @@ private Literal(Object value, DataType dataType) { *
  • TIMESTAMP: {@link Long} represents the microseconds since epoch in UTC *
  • TIMESTAMP_NTZ: {@link Long} represents the microseconds since epoch with no timezone *
  • DECIMAL: {@link BigDecimal}.Use {@link #getDataType()} to find the precision and scale + *
  • GEOMETRY: {@link String} WKT (Well-Known Text) representation + *
  • GEOGRAPHY: {@link String} WKT (Well-Known Text) representation * * * @return Literal value. diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/expressions/StGeometryBoxesIntersect.java b/kernel/kernel-api/src/main/java/io/delta/kernel/expressions/StGeometryBoxesIntersect.java new file mode 100644 index 00000000000..afceea94e38 --- /dev/null +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/expressions/StGeometryBoxesIntersect.java @@ -0,0 +1,66 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.kernel.expressions; + +import static io.delta.kernel.internal.util.Preconditions.checkArgument; + +import io.delta.kernel.types.GeometryType; +import java.util.Arrays; + +/** + * A predicate that tests whether the bounding box of a geometry column intersects a given query + * bounding box. + * + *

    This expression is not directly evaluatable. It is transformed to an internal + * ST_GEOMETRY_BOXES_INTERSECT_ON_STATS predicate during data-skipping filter construction, which + * evaluates the intersection of the column's min/max bounding box against the query box. + * + * @since 4.0.0 + */ +public class StGeometryBoxesIntersect extends Predicate { + + public static final String NAME = "ST_GEOMETRY_BOXES_INTERSECT"; + + /** + * @param column the geometry column to test + * @param queryMin lower-left corner of the query bounding box as a GeometryType WKT POINT literal + * @param queryMax upper-right corner of the query bounding box as a GeometryType WKT POINT + * literal + */ + public StGeometryBoxesIntersect(Column column, Literal queryMin, Literal queryMax) { + super(NAME, Arrays.asList(column, queryMin, queryMax)); + checkArgument( + queryMin.getDataType() instanceof GeometryType, + "queryMin must be a GeometryType WKT literal, got: %s", + queryMin.getDataType()); + checkArgument( + queryMax.getDataType() instanceof GeometryType, + "queryMax must be a GeometryType WKT literal, got: %s", + queryMax.getDataType()); + } + + public Column getColumn() { + return (Column) getChildren().get(0); + } + + public Literal getQueryMin() { + return (Literal) getChildren().get(1); + } + + public Literal getQueryMax() { + return (Literal) getChildren().get(2); + } +} diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/DeltaErrors.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/DeltaErrors.java index 5849ea8e1e0..ca3d5a5ec36 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/DeltaErrors.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/DeltaErrors.java @@ -536,6 +536,17 @@ public static KernelException cannotModifyAppendOnlyTable(String tablePath) { tablePath, TableConfig.APPEND_ONLY_ENABLED.getKey())); } + public static KernelException cdfMixedAddRemoveNotSupported(String tablePath) { + return new KernelException( + String.format( + "Cannot add and remove data in the same transaction when Change Data Feed is enabled " + + "on table %s. This would require writing CDC files for DML operations, which is " + + "not yet supported by Delta Kernel. You can perform add-only operations (like " + + "INSERT or CREATE TABLE), remove-only operations (like DELETE), or mixed " + + "operations with dataChange=false (like OPTIMIZE).", + tablePath)); + } + public static KernelException rowTrackingMetadataMissingInFile(String entry, String filePath) { return new KernelException( String.format("Required metadata key %s is not present in scan file %s.", entry, filePath)); diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TableConfig.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TableConfig.java index ffb695526eb..253ba7da5ea 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TableConfig.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TableConfig.java @@ -367,7 +367,7 @@ public static class UniversalFormats { /** * Table property that enables modifying the table in accordance with the Delta-Variant Shredding - * Preview protocol. + * protocol. * * @see diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TransactionBuilderImpl.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TransactionBuilderImpl.java index 26443fdfcdc..ceb72b90921 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TransactionBuilderImpl.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TransactionBuilderImpl.java @@ -29,6 +29,7 @@ import io.delta.kernel.expressions.Column; import io.delta.kernel.internal.actions.*; import io.delta.kernel.internal.commit.DefaultFileSystemManagedTableOnlyCommitter; +import io.delta.kernel.internal.rowtracking.MaterializedRowTrackingColumn; import io.delta.kernel.internal.tablefeatures.TableFeatures; import io.delta.kernel.types.StructType; import java.util.*; @@ -228,9 +229,8 @@ protected TransactionImpl buildTransactionInternal( || enablesDomainMetadataSupport; // domain metadata support added if (!needsMetadataOrProtocolUpdate) { - // TODO: fix this https://github.com/delta-io/delta/issues/4713 // Return early if there is no metadata or protocol updates and isCreateOrReplace=false - new TransactionImpl( + return new TransactionImpl( false, // isCreateOrReplace table.getDataPath(), latestSnapshot, @@ -314,7 +314,8 @@ protected TransactionImpl buildTransactionInternal( * Validates that Kernel can write to the existing table with the latest snapshot as provided. * This means (1) Kernel supports the reader and writer protocol of the table (2) if a transaction * identifier has been provided in this txn builder, a concurrent write has not already committed - * this transaction (3) Updating a partitioned table with clustering columns is not allowed. + * this transaction (3) Updating a partitioned table with clustering columns is not allowed (4) + * Row tracking configs are present when row tracking is enabled. */ protected void validateWriteToExistingTable( Engine engine, SnapshotImpl snapshot, boolean isCreateOrReplace) { @@ -338,6 +339,12 @@ protected void validateWriteToExistingTable( snapshot.getMetadata().getPartitionColNames(), inputLogicalClusteringColumns.get()); } + // Validate row tracking configs are present when row tracking is enabled. This must run + // on every write, including the early-return path that skips TransactionMetadataFactory. + if (!isCreateOrReplace) { + MaterializedRowTrackingColumn.validateRowTrackingConfigsNotMissing( + snapshot.getMetadata(), table.getPath(engine)); + } } /** diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TransactionImpl.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TransactionImpl.java index f85f156bf9b..0851dc7d480 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TransactionImpl.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TransactionImpl.java @@ -68,6 +68,7 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.util.*; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -542,6 +543,13 @@ private Tuple2> doCommit( } boolean isAppendOnlyTable = APPEND_ONLY_ENABLED.fromMetadata(metadata); + boolean isCdfEnabled = TableConfig.CHANGE_DATA_FEED_ENABLED.fromMetadata(metadata); + + // Track CDF validation state: whether we've seen adds and removes + // with enableChangeDataFeed=true + // This is wrapped in an AtomicBoolean to allow modification from within the lambda + final AtomicBoolean hasAddWithDataChange = new AtomicBoolean(false); + final AtomicBoolean hasRemoveWithDataChange = new AtomicBoolean(false); // Create a new CloseableIterator that will return the metadata actions followed by the // data actions. @@ -556,6 +564,21 @@ private Tuple2> doCommit( if (isAppendOnlyTable && removeFile.getDataChange()) { throw DeltaErrors.cannotModifyAppendOnlyTable(dataPath.toString()); } + if (isCdfEnabled && removeFile.getDataChange()) { + hasRemoveWithDataChange.set(true); + if (hasAddWithDataChange.get()) { + throw DeltaErrors.cdfMixedAddRemoveNotSupported(dataPath.toString()); + } + } + } + if (!action.isNullAt(ADD_FILE_ORDINAL)) { + AddFile addFile = new AddFile(action.getStruct(ADD_FILE_ORDINAL)); + if (isCdfEnabled && addFile.getDataChange()) { + hasAddWithDataChange.set(true); + if (hasRemoveWithDataChange.get()) { + throw DeltaErrors.cdfMixedAddRemoveNotSupported(dataPath.toString()); + } + } } return action; }); diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TransactionMetadataFactory.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TransactionMetadataFactory.java index 90ffb2a9d00..1f91cc18813 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TransactionMetadataFactory.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/TransactionMetadataFactory.java @@ -498,12 +498,18 @@ private void performIcebergCompatUpgradesAndValidation() { oldMetadata.getConfiguration(), metadata.getConfiguration())); }); + // Pass the previous protocol to IcebergCompat checks as defense-in-depth: if the read + // snapshot already had deletion vectors enabled, the check should reject it even when the + // new protocol in this transaction does not include DVs. The conflict checker may also + // catch this at commit time, but checking early gives a clearer error message. + Optional prevProtocol = latestSnapshotOpt.map(s -> s.getProtocol()); + // We must do our icebergWriterCompatV1 checks/updates FIRST since it has stricter column // mapping requirements (id mode) than icebergCompatV2. It also may enable icebergCompatV2. Optional icebergWriterCompatV1 = IcebergWriterCompatV1MetadataValidatorAndUpdater .validateAndUpdateIcebergWriterCompatV1Metadata( - isCreateOrReplace, getEffectiveMetadata(), getEffectiveProtocol()); + isCreateOrReplace, getEffectiveMetadata(), getEffectiveProtocol(), prevProtocol); if (icebergWriterCompatV1.isPresent()) { newMetadata = icebergWriterCompatV1; } @@ -511,7 +517,7 @@ private void performIcebergCompatUpgradesAndValidation() { Optional icebergWriterCompatV3 = IcebergWriterCompatV3MetadataValidatorAndUpdater .validateAndUpdateIcebergWriterCompatV3Metadata( - isCreateOrReplace, getEffectiveMetadata(), getEffectiveProtocol()); + isCreateOrReplace, getEffectiveMetadata(), getEffectiveProtocol(), prevProtocol); if (icebergWriterCompatV3.isPresent()) { newMetadata = icebergWriterCompatV3; } @@ -519,13 +525,13 @@ private void performIcebergCompatUpgradesAndValidation() { // TODO: refactor this method to use a single validator and updater. Optional icebergCompatV2Metadata = IcebergCompatV2MetadataValidatorAndUpdater.validateAndUpdateIcebergCompatV2Metadata( - isCreateOrReplace, getEffectiveMetadata(), getEffectiveProtocol()); + isCreateOrReplace, getEffectiveMetadata(), getEffectiveProtocol(), prevProtocol); if (icebergCompatV2Metadata.isPresent()) { newMetadata = icebergCompatV2Metadata; } Optional icebergCompatV3Metadata = IcebergCompatV3MetadataValidatorAndUpdater.validateAndUpdateIcebergCompatV3Metadata( - isCreateOrReplace, getEffectiveMetadata(), getEffectiveProtocol()); + isCreateOrReplace, getEffectiveMetadata(), getEffectiveProtocol(), prevProtocol); if (icebergCompatV3Metadata.isPresent()) { newMetadata = icebergCompatV3Metadata; } diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/DeletionVectorDescriptor.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/DeletionVectorDescriptor.java index 30d457a9f02..5691ec08bf9 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/DeletionVectorDescriptor.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/DeletionVectorDescriptor.java @@ -204,7 +204,7 @@ public String serializeToBase64() { } public boolean isInline() { - return storageType == INLINE_DV_MARKER; + return INLINE_DV_MARKER.equals(storageType); } public boolean isOnDisk() { diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/GenerateIcebergCompatActionUtils.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/GenerateIcebergCompatActionUtils.java index 01cd87291ec..f464e1adca4 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/GenerateIcebergCompatActionUtils.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions/GenerateIcebergCompatActionUtils.java @@ -306,8 +306,8 @@ public static Row generateIcebergCompatWriterV3RemoveAction( fileStatus, partitionValues, dataChange, - Optional.empty() /* baseRowId */, - Optional.empty() /* defaultRowCommitVersion */, + baseRowId, + defaultRowCommitVersion, deletionVectorDescriptor); return SingleAction.createRemoveFileSingleAction(removeFileRow); } diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/checksum/CRCInfo.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/checksum/CRCInfo.java index fbb69fcf160..a4ee453a69a 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/checksum/CRCInfo.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/checksum/CRCInfo.java @@ -51,6 +51,7 @@ public class CRCInfo { private static final String TXN_ID = "txnId"; private static final String DOMAIN_METADATA = "domainMetadata"; private static final String FILE_SIZE_HISTOGRAM = "fileSizeHistogram"; + private static final String HISTOGRAM_OPT = "histogramOpt"; public static final StructType CRC_FILE_SCHEMA = new StructType() @@ -64,6 +65,10 @@ public class CRCInfo { .add(DOMAIN_METADATA, new ArrayType(DomainMetadata.FULL_SCHEMA, false), /*nullable*/ true) .add(FILE_SIZE_HISTOGRAM, FileSizeHistogram.FULL_SCHEMA, /*nullable*/ true); + // Used by ChecksumReader to support reading CRC files with the legacy "histogramOpt" field. + public static final StructType CRC_FILE_READ_SCHEMA = + CRC_FILE_SCHEMA.add(HISTOGRAM_OPT, FileSizeHistogram.FULL_SCHEMA, /*nullable*/ true); + public static Optional fromColumnarBatch( long version, ColumnarBatch batch, int rowId, String crcFilePath) { // Read required fields. @@ -89,6 +94,13 @@ public static Optional fromColumnarBatch( Optional fileSizeHistogram = FileSizeHistogram.fromColumnVector( batch.getColumnVector(getSchemaIndex(FILE_SIZE_HISTOGRAM)), rowId); + if (!fileSizeHistogram.isPresent()) { + int histogramOptIdx = batch.getSchema().indexOf(HISTOGRAM_OPT); + if (histogramOptIdx >= 0) { + fileSizeHistogram = + FileSizeHistogram.fromColumnVector(batch.getColumnVector(histogramOptIdx), rowId); + } + } ColumnVector domainMetadataVector = batch.getColumnVector(getSchemaIndex(DOMAIN_METADATA)); Optional> domainMetadata = domainMetadataVector.isNullAt(rowId) diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/checksum/ChecksumReader.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/checksum/ChecksumReader.java index 4a22a831f15..86faf00bc00 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/checksum/ChecksumReader.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/checksum/ChecksumReader.java @@ -45,7 +45,7 @@ public static Optional tryReadChecksumFile(Engine engine, FileStatus ch .getJsonHandler() .readJsonFiles( singletonCloseableIterator(checkSumFile), - CRCInfo.CRC_FILE_SCHEMA, + CRCInfo.CRC_FILE_READ_SCHEMA, Optional.empty())) { // We do this instead of iterating through the rows or using `getSingularRow` so we // can use the existing fromColumnVector methods in Protocol, Metadata, Format etc diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergCompatMetadataValidatorAndUpdater.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergCompatMetadataValidatorAndUpdater.java index 0125c48390e..cd64947fb51 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergCompatMetadataValidatorAndUpdater.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergCompatMetadataValidatorAndUpdater.java @@ -78,21 +78,28 @@ public static class IcebergCompatInputContext { final boolean isCreatingNewTable; final Metadata newMetadata; final Protocol newProtocol; + /** + * The protocol from the previous snapshot, used to guard against concurrent writers that may + * have enabled incompatible features (e.g. deletion vectors). Empty for new tables. + */ + final Optional prevProtocol; public IcebergCompatInputContext( String compatFeatureName, boolean isCreatingNewTable, Metadata newMetadata, - Protocol newProtocol) { + Protocol newProtocol, + Optional prevProtocol) { this.compatFeatureName = compatFeatureName; this.isCreatingNewTable = isCreatingNewTable; this.newMetadata = newMetadata; this.newProtocol = newProtocol; + this.prevProtocol = prevProtocol; } public IcebergCompatInputContext withUpdatedMetadata(Metadata newMetadata) { return new IcebergCompatInputContext( - compatFeatureName, isCreatingNewTable, newMetadata, newProtocol); + compatFeatureName, isCreatingNewTable, newMetadata, newProtocol, prevProtocol); } } @@ -319,7 +326,18 @@ protected static IcebergCompatCheck hasOnlySupportedTypes( protected static final IcebergCompatCheck CHECK_HAS_NO_DELETION_VECTORS = (inputContext) -> { - if (inputContext.newProtocol.supportsFeature(DELETION_VECTORS_RW_FEATURE)) { + // Check both newProtocol and prevProtocol as defense-in-depth against cases where + // the previous snapshot already had deletion vectors enabled. This matches Spark's + // CheckDeletionVectorDisabled which checks both prevSnapshot and newestProtocol. + // Note: the conflict checker may also catch concurrent DV enablement at commit time. + boolean dvInNewProtocol = + inputContext.newProtocol.supportsFeature(DELETION_VECTORS_RW_FEATURE); + boolean dvInPrevProtocol = + inputContext + .prevProtocol + .map(prev -> prev.supportsFeature(DELETION_VECTORS_RW_FEATURE)) + .orElse(false); + if (dvInNewProtocol || dvInPrevProtocol) { throw DeltaErrors.icebergCompatIncompatibleTableFeatures( inputContext.compatFeatureName, Collections.singleton(DELETION_VECTORS_RW_FEATURE)); } diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergCompatV2MetadataValidatorAndUpdater.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergCompatV2MetadataValidatorAndUpdater.java index 7619792d1a0..12eda095b94 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergCompatV2MetadataValidatorAndUpdater.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergCompatV2MetadataValidatorAndUpdater.java @@ -42,10 +42,17 @@ public class IcebergCompatV2MetadataValidatorAndUpdater * requirements */ public static Optional validateAndUpdateIcebergCompatV2Metadata( - boolean isCreatingNewTable, Metadata newMetadata, Protocol newProtocol) { + boolean isCreatingNewTable, + Metadata newMetadata, + Protocol newProtocol, + Optional prevProtocol) { return INSTANCE.validateAndUpdateMetadata( new IcebergCompatInputContext( - INSTANCE.compatFeatureName(), isCreatingNewTable, newMetadata, newProtocol)); + INSTANCE.compatFeatureName(), + isCreatingNewTable, + newMetadata, + newProtocol, + prevProtocol)); } /** diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergCompatV3MetadataValidatorAndUpdater.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergCompatV3MetadataValidatorAndUpdater.java index df7857a3672..9e5b3e32e94 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergCompatV3MetadataValidatorAndUpdater.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergCompatV3MetadataValidatorAndUpdater.java @@ -64,10 +64,17 @@ public static void validateIcebergCompatV3Change( * requirements */ public static Optional validateAndUpdateIcebergCompatV3Metadata( - boolean isCreatingNewTable, Metadata newMetadata, Protocol newProtocol) { + boolean isCreatingNewTable, + Metadata newMetadata, + Protocol newProtocol, + Optional prevProtocol) { return INSTANCE.validateAndUpdateMetadata( new IcebergCompatInputContext( - INSTANCE.compatFeatureName(), isCreatingNewTable, newMetadata, newProtocol)); + INSTANCE.compatFeatureName(), + isCreatingNewTable, + newMetadata, + newProtocol, + prevProtocol)); } /** diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV1MetadataValidatorAndUpdater.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV1MetadataValidatorAndUpdater.java index 7f7b947bc58..408da7eec98 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV1MetadataValidatorAndUpdater.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV1MetadataValidatorAndUpdater.java @@ -93,10 +93,17 @@ public static void validateIcebergWriterCompatV1Change( * requirements */ public static Optional validateAndUpdateIcebergWriterCompatV1Metadata( - boolean isCreatingNewTable, Metadata newMetadata, Protocol newProtocol) { + boolean isCreatingNewTable, + Metadata newMetadata, + Protocol newProtocol, + Optional prevProtocol) { return INSTANCE.validateAndUpdateMetadata( new IcebergCompatInputContext( - INSTANCE.compatFeatureName(), isCreatingNewTable, newMetadata, newProtocol)); + INSTANCE.compatFeatureName(), + isCreatingNewTable, + newMetadata, + newProtocol, + prevProtocol)); } /// ////////////////////////////////////////////////////////////////////////////// @@ -119,7 +126,8 @@ public static Optional validateAndUpdateIcebergWriterCompatV1Metadata( .validateAndUpdateIcebergCompatV2Metadata( inputContext.isCreatingNewTable, inputContext.newMetadata, - inputContext.newProtocol)); + inputContext.newProtocol, + inputContext.prevProtocol)); /** * Current set of allowed table features for Iceberg writer compat V1. This combines the common diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV3MetadataValidatorAndUpdater.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV3MetadataValidatorAndUpdater.java index aaef34cdb6a..3586d5bcf2d 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV3MetadataValidatorAndUpdater.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV3MetadataValidatorAndUpdater.java @@ -64,10 +64,17 @@ public static void validateIcebergWriterCompatV3Change( * requirements */ public static Optional validateAndUpdateIcebergWriterCompatV3Metadata( - boolean isCreatingNewTable, Metadata newMetadata, Protocol newProtocol) { + boolean isCreatingNewTable, + Metadata newMetadata, + Protocol newProtocol, + Optional prevProtocol) { return INSTANCE.validateAndUpdateMetadata( new IcebergCompatInputContext( - INSTANCE.compatFeatureName(), isCreatingNewTable, newMetadata, newProtocol)); + INSTANCE.compatFeatureName(), + isCreatingNewTable, + newMetadata, + newProtocol, + prevProtocol)); } /// ////////////////////////////////////////////////////////////////////////////// @@ -90,7 +97,8 @@ public static Optional validateAndUpdateIcebergWriterCompatV3Metadata( .validateAndUpdateIcebergCompatV3Metadata( inputContext.isCreatingNewTable, inputContext.newMetadata, - inputContext.newProtocol)); + inputContext.newProtocol, + inputContext.prevProtocol)); /** * Current set of allowed table features for Iceberg writer compat V3. This combines the common @@ -105,6 +113,7 @@ public static Optional validateAndUpdateIcebergWriterCompatV3Metadata( ICEBERG_WRITER_COMPAT_V3, DELETION_VECTORS_RW_FEATURE, VARIANT_RW_FEATURE, + VARIANT_SHREDDING_RW_FEATURE, VARIANT_SHREDDING_PREVIEW_RW_FEATURE, VARIANT_RW_PREVIEW_FEATURE, // Also allow writerV1 features for backward compatibility. diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/skipping/DataSkippingUtils.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/skipping/DataSkippingUtils.java index 11d01a5e36c..37a644d1091 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/skipping/DataSkippingUtils.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/skipping/DataSkippingUtils.java @@ -286,6 +286,31 @@ private static Optional constructDataSkippingFilter( return constructNotDataSkippingFilters( asPredicate(getUnaryChild(dataFilters)), schemaHelper); + case StGeometryBoxesIntersect.NAME: + if (dataFilters instanceof StGeometryBoxesIntersect) { + StGeometryBoxesIntersect stExpr = (StGeometryBoxesIntersect) dataFilters; + Column col = stExpr.getColumn(); + if (schemaHelper.isSkippingEligibleMinMaxColumn(col)) { + Column minCol = schemaHelper.getMinColumn(col, Optional.empty())._1; + Column maxCol = schemaHelper.getMaxColumn(col, Optional.empty())._1; + Literal queryMin = stExpr.getQueryMin(); + Literal queryMax = stExpr.getQueryMax(); + Set refCols = + new HashSet() { + { + add(minCol); + add(maxCol); + } + }; + return Optional.of( + new DataSkippingPredicate( + "ST_GEOMETRY_BOXES_INTERSECT_ON_STATS", + Arrays.asList(minCol, maxCol, queryMin, queryMax), + refCols)); + } + } + break; + // TODO more expressions } return Optional.empty(); diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/skipping/StatsSchemaHelper.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/skipping/StatsSchemaHelper.java index 804f9280192..1cc6d6c0bcf 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/skipping/StatsSchemaHelper.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/skipping/StatsSchemaHelper.java @@ -69,7 +69,11 @@ public static boolean isSkippingEligibleDataType(DataType dataType) { || // StringType is eligible, but since its string can include collation info, it needs to // be matched separately. - dataType instanceof StringType; + dataType instanceof StringType + // Geometry/geography types are eligible but need separate matching + // due to SRID (and algorithm) parameters + || dataType instanceof GeometryType + || dataType instanceof GeographyType; } /** diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/tablefeatures/TableFeatures.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/tablefeatures/TableFeatures.java index 99f61f14450..ff52106c0ec 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/tablefeatures/TableFeatures.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/tablefeatures/TableFeatures.java @@ -143,8 +143,10 @@ private static class ChangeDataFeedFeature extends TableFeature.LegacyWriterFeat @Override public boolean hasKernelWriteSupport(Metadata metadata) { - // writable if change data feed is disabled - return !TableConfig.CHANGE_DATA_FEED_ENABLED.fromMetadata(metadata); + // Kernel now supports writes to CDF-enabled tables with restrictions. + // Validation is performed at commit time to check for unsupported mixed operations + // (add+remove with dataChange=true) which would require CDC file generation. + return true; } @Override @@ -203,6 +205,20 @@ public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata me } } + static final TableFeature GEOSPATIAL_RW_FEATURE = new GeoSpatialTableFeature(); + + private static class GeoSpatialTableFeature extends TableFeature.ReaderWriterFeature + implements FeatureAutoEnabledByMetadata { + GeoSpatialTableFeature() { + super("geospatial", /* minReaderVersion = */ 3, /* minWriterVersion = */ 7); + } + + @Override + public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata metadata) { + return hasGeospatial(metadata); + } + } + /* ---- Start: variantType ---- */ // Base class for variantType and variantType-preview features. Both features are same in terms // of behavior and given the feature is graduated, we will enable the `variantType` by default @@ -238,27 +254,49 @@ public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata me new VariantTypeTableFeatureBase("variantType-preview"); /* ---- End: variantType ---- */ - /* ---- Start: variantShredding-preview ---- */ - public static final TableFeature VARIANT_SHREDDING_PREVIEW_RW_FEATURE = - new VariantShreddingPreviewFeature(); - - private static class VariantShreddingPreviewFeature extends TableFeature.ReaderWriterFeature - implements FeatureAutoEnabledByMetadata { - VariantShreddingPreviewFeature() { - super("variantShredding-preview", /* minReaderVersion = */ 3, /* minWriterVersion = */ 7); + /* ---- Start: variantShredding ---- */ + // Base class for variantShredding and variantShredding-preview features. Both features have + // identical behavior. Now that variantShredding has graduated to GA: + // + // - When `delta.enableVariantShredding` is set to true, the GA feature (`variantShredding`) + // is automatically enabled unless the table already has `variantShredding-preview` in its + // protocol (to avoid breaking clients that only understand the preview feature). + // - The preview feature (`variantShredding-preview`) is never auto-enabled. To use it on a + // new table, a user must explicitly set `delta.feature.variantShredding-preview=supported` + // in the table properties. + private static class VariantShreddingTableFeatureBase extends TableFeature.ReaderWriterFeature { + VariantShreddingTableFeatureBase(String featureName) { + super(featureName, /* minReaderVersion = */ 3, /* minWriterVersion = */ 7); } @Override - public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata metadata) { - return TableConfig.VARIANT_SHREDDING_ENABLED.fromMetadata(metadata); + public Set requiredFeatures() { + return new HashSet<>(Arrays.asList(TableFeatures.VARIANT_RW_FEATURE)); + } + } + + private static class VariantShreddingTableFeature extends VariantShreddingTableFeatureBase + implements FeatureAutoEnabledByMetadata { + VariantShreddingTableFeature() { + super("variantShredding"); } @Override - public Set requiredFeatures() { - return new HashSet<>(Arrays.asList(TableFeatures.VARIANT_RW_FEATURE)); + public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata metadata) { + return TableConfig.VARIANT_SHREDDING_ENABLED.fromMetadata(metadata) + && + // Don't automatically enable the stable feature if the preview feature is + // already supported, to avoid possibly breaking old clients that only + // support the preview feature. + !protocol.supportsFeature(VARIANT_SHREDDING_PREVIEW_RW_FEATURE); } } - /* ---- End: variantShredding-preview ---- */ + + public static final TableFeature VARIANT_SHREDDING_RW_FEATURE = + new VariantShreddingTableFeature(); + public static final TableFeature VARIANT_SHREDDING_PREVIEW_RW_FEATURE = + new VariantShreddingTableFeatureBase("variantShredding-preview"); + /* ---- End: variantShredding ---- */ public static final TableFeature DOMAIN_METADATA_W_FEATURE = new DomainMetadataFeature(); @@ -516,6 +554,34 @@ private static class MaterializePartitionColumnsFeature extends TableFeature.Wri } } + /** This feature was replaced with its stable version, `collations`. */ + public static final TableFeature COLLATIONS_PREVIEW_W_FEATURE = new CollationsPreview(); + + private static class CollationsPreview extends TableFeature.WriterFeature { + CollationsPreview() { + super("collations-preview", /* minWriterVersion = */ 7); + } + } + + /** The stable collation feature. */ + public static final TableFeature COLLATIONS_W_FEATURE = new Collations(); + + private static class Collations extends TableFeature.WriterFeature + implements FeatureAutoEnabledByMetadata { + Collations() { + super("collations", /* minWriterVersion = */ 7); + } + + @Override + public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata metadata) { + return hasCollatedColumn(metadata.getSchema()) + && + // Don't automatically enable the stable feature if the preview feature is already + // supported, to avoid breaking old clients that only support the preview feature. + !protocol.supportsFeature(COLLATIONS_PREVIEW_W_FEATURE); + } + } + ///////////////////////////////////////////////////////////////////////////////// /// END: Define the {@link TableFeature}s /// ///////////////////////////////////////////////////////////////////////////////// @@ -557,9 +623,13 @@ private static class MaterializePartitionColumnsFeature extends TableFeature.Wri VACUUM_PROTOCOL_CHECK_RW_FEATURE, VARIANT_RW_FEATURE, VARIANT_RW_PREVIEW_FEATURE, + VARIANT_SHREDDING_RW_FEATURE, VARIANT_SHREDDING_PREVIEW_RW_FEATURE, ICEBERG_WRITER_COMPAT_V1, - ICEBERG_WRITER_COMPAT_V3)); + ICEBERG_WRITER_COMPAT_V3, + COLLATIONS_PREVIEW_W_FEATURE, + COLLATIONS_W_FEATURE, + GEOSPATIAL_RW_FEATURE)); public static final Map TABLE_FEATURE_MAP = Collections.unmodifiableMap( @@ -857,6 +927,15 @@ private static Set getDependencyFeatures(Set feature } } + public static boolean hasGeospatial(Metadata metadata) { + return new SchemaIterable(metadata.getSchema()) + .stream() + .anyMatch( + element -> + element.getField().getDataType() instanceof GeometryType + || element.getField().getDataType() instanceof GeographyType); + } + /** * Check if the table schema has a column of type. Caution: works only for the primitive types. */ @@ -864,4 +943,18 @@ private static boolean hasTypeColumn(StructType tableSchema, DataType type) { return new SchemaIterable(tableSchema) .stream().anyMatch(element -> element.getField().getDataType().equals(type)); } + + /** + * Check if the table schema has any string column with a non-default (non-UTF8_BINARY) collation. + */ + static boolean hasCollatedColumn(StructType tableSchema) { + return new SchemaIterable(tableSchema) + .stream() + .anyMatch( + element -> { + DataType dataType = element.getField().getDataType(); + return dataType instanceof StringType + && !((StringType) dataType).isUTF8BinaryCollated(); + }); + } } diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/GeometryUtils.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/GeometryUtils.java new file mode 100644 index 00000000000..c784f1b6e63 --- /dev/null +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/GeometryUtils.java @@ -0,0 +1,178 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.kernel.internal.util; + +import java.util.Objects; +import java.util.OptionalDouble; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** Utility for parsing and formatting WKT POINT strings for geometry/geography stats. */ +public class GeometryUtils { + private GeometryUtils() {} + + // Matches: POINT [ZM|Z|M] (coords) - (case-insensitive, flexible whitespace) + private static final Pattern POINT_WKT_PATTERN = + Pattern.compile("^POINT\\s*(ZM|Z|M)?\\s*\\(([^)]+)\\)$", Pattern.CASE_INSENSITIVE); + + // Matches: POINT [Z|M|ZM] EMPTY - (case-insensitive, flexible whitespace) + private static final Pattern POINT_EMPTY_PATTERN = + Pattern.compile("^POINT\\s*(ZM|Z|M)?\\s+EMPTY$", Pattern.CASE_INSENSITIVE); + + /** + * Formats x/y (and optional z/m) coordinates as a WKT POINT string. Uses standard OGC WKT format: + * "POINT(1.0 2.0)", "POINT Z (1.0 2.0 3.0)", "POINT ZM (1.0 2.0 3.0 4.0)". + */ + public static String formatPointWKT(double x, double y, OptionalDouble z, OptionalDouble m) { + boolean hasZ = z.isPresent(); + boolean hasM = m.isPresent(); + if (Double.isNaN(x) && Double.isNaN(y)) { + StringBuilder sb = new StringBuilder("POINT"); + if (hasZ == hasM) { + if (hasZ) sb.append(" ZM"); + } else { + sb.append(hasZ ? " Z" : " M"); + } + return sb.append(" EMPTY").toString(); + } + StringBuilder sb = new StringBuilder("POINT"); + if (hasZ == hasM) { + if (hasZ) sb.append(" ZM "); + } else { + sb.append(hasZ ? " Z " : " M "); + } + sb.append("(").append(x).append(" ").append(y); + z.ifPresent(v -> sb.append(" ").append(v)); + m.ifPresent(v -> sb.append(" ").append(v)); + return sb.append(")").toString(); + } + + /** + * Validates that the given string is a well-formed WKT POINT. Throws KernelException if the + * string is null or not a valid POINT WKT. + */ + public static void validatePointWKT(String wkt) { + try { + parsePointXY(wkt); + } catch (IllegalArgumentException e) { + throw new io.delta.kernel.exceptions.KernelException( + String.format("Geospatial stats must be a valid POINT WKT but got: %s", wkt), e); + } + } + + /** + * Parses only the x and y coordinates from a WKT POINT string. Returns double[]{x, y}. Z and M + * coordinates are validated but not returned because stats bounding boxes only use 2D + * coordinates. + * + *

    Supported formats: POINT(x y), POINT Z (x y z), POINT M (x y m), POINT ZM (x y z m), POINT + * EMPTY, POINT Z EMPTY, POINT M EMPTY, POINT ZM EMPTY. + * + *

    All EMPTY variants return {NaN, NaN}. + * + * @throws IllegalArgumentException if not a valid POINT WKT + */ + public static double[] parsePointXY(String wkt) { + Objects.requireNonNull(wkt, "WKT POINT string cannot be null"); + String trimmed = wkt.trim(); + + if (POINT_EMPTY_PATTERN.matcher(trimmed).matches()) { + return new double[] {Double.NaN, Double.NaN}; + } + + Matcher matcher = POINT_WKT_PATTERN.matcher(trimmed); + if (!matcher.matches()) { + throw new IllegalArgumentException("Invalid WKT POINT string: " + wkt); + } + + String modifier = matcher.group(1); + String coordStr = matcher.group(2).trim(); + String[] parts = coordStr.split("\\s+"); + + String mod = (modifier == null) ? "" : modifier.toUpperCase(); + int expectedCount; + switch (mod) { + case "ZM": + expectedCount = 4; + break; + case "Z": + case "M": + expectedCount = 3; + break; + default: + expectedCount = 2; + break; + } + + if (parts.length != expectedCount) { + throw new IllegalArgumentException( + String.format( + "POINT %s expects %d coordinates but got %d: %s", + mod, expectedCount, parts.length, wkt)); + } + + try { + double x = Double.parseDouble(parts[0]); + double y = Double.parseDouble(parts[1]); + if (!Double.isFinite(x) || !Double.isFinite(y)) { + throw new IllegalArgumentException("POINT coordinates must be finite numbers: " + wkt); + } + for (int i = 2; i < parts.length; i++) { + double v = Double.parseDouble(parts[i]); + if (!Double.isFinite(v)) { + throw new IllegalArgumentException("POINT coordinates must be finite numbers: " + wkt); + } + } + return new double[] {x, y}; + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid coordinate in WKT POINT string: " + wkt, e); + } + } + + /** + * Validates that the given WKT POINT is a valid geography point with coordinates in range: + * longitude in [-180, 180] and latitude in [-90, 90]. Throws KernelException if invalid. + */ + public static void validateGeographyPointWKT(String wkt) { + double[] xy; + try { + xy = parsePointXY(wkt); + } catch (IllegalArgumentException e) { + throw new io.delta.kernel.exceptions.KernelException( + String.format("Geospatial stats must be a valid POINT WKT" + " but got: %s", wkt), e); + } + validateGeographyCoordinates(xy[0], xy[1], wkt); + } + + /** + * Validates that longitude is in [-180, 180] and latitude is in [-90, 90]. Throws KernelException + * if out of range. + */ + public static void validateGeographyCoordinates(double lon, double lat, String wkt) { + // POINT EMPTY produces NaN — skip range check + if (Double.isNaN(lon) && Double.isNaN(lat)) { + return; + } + if (lon < -180 || lon > 180 || lat < -90 || lat > 90) { + throw new io.delta.kernel.exceptions.KernelException( + String.format( + "Geography coordinates out of range: longitude" + + " must be in [-180, 180], latitude must be" + + " in [-90, 90], but got: %s", + wkt)); + } + } +} diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/JsonUtils.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/JsonUtils.java index 3104c57f585..968df6b2b2c 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/JsonUtils.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/JsonUtils.java @@ -258,6 +258,19 @@ public static Literal parseJsonValueToLiteral(JsonNode valueNode, DataType dataT } String textValue = valueNode.asText(); return Literal.ofString(textValue); + } else if (dataType instanceof GeometryType || dataType instanceof GeographyType) { + if (!valueNode.isTextual()) { + String typeName = dataType instanceof GeometryType ? "geometry" : "geography"; + throw new KernelException( + String.format("Expected %s as WKT string but got: %s", typeName, valueNode)); + } + String wkt = valueNode.asText(); + if (dataType instanceof GeographyType) { + GeometryUtils.validateGeographyPointWKT(wkt); + } else { + GeometryUtils.validatePointWKT(wkt); + } + return Literal.ofGeospatialWKT(wkt, dataType); } else { throw unsupportedStatsDataType(dataType); } diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/SchemaUtils.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/SchemaUtils.java index 4aca1ae22ec..0e500265c67 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/SchemaUtils.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/SchemaUtils.java @@ -853,7 +853,9 @@ protected static void validateSupportedType(DataType dataType) { || dataType instanceof DateType || dataType instanceof TimestampType || dataType instanceof TimestampNTZType - || dataType instanceof VariantType) { + || dataType instanceof VariantType + || dataType instanceof GeometryType + || dataType instanceof GeographyType) { // supported types return; } else if (dataType instanceof StructType) { diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/statistics/DataFileStatistics.java b/kernel/kernel-api/src/main/java/io/delta/kernel/statistics/DataFileStatistics.java index 84e77904f33..f09cda1c665 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/statistics/DataFileStatistics.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/statistics/DataFileStatistics.java @@ -25,6 +25,7 @@ import io.delta.kernel.expressions.Literal; import io.delta.kernel.internal.DeltaErrors; import io.delta.kernel.internal.skipping.StatsSchemaHelper; +import io.delta.kernel.internal.util.GeometryUtils; import io.delta.kernel.internal.util.JsonUtils; import io.delta.kernel.types.*; import java.io.IOException; @@ -437,6 +438,12 @@ private void writeJsonValue(JsonGenerator generator, Literal literal) throws IOE LocalDateTime localDateTime = ChronoUnit.MICROS.addTo(EPOCH, epochMicros).toLocalDateTime(); LocalDateTime truncated = localDateTime.truncatedTo(ChronoUnit.MILLIS); generator.writeString(truncated.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME)); + } else if (type instanceof GeometryType) { + GeometryUtils.validatePointWKT((String) value); + generator.writeString((String) value); + } else if (type instanceof GeographyType) { + GeometryUtils.validateGeographyPointWKT((String) value); + generator.writeString((String) value); } else { throw unsupportedStatsDataType(type); } diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/types/FieldMetadata.java b/kernel/kernel-api/src/main/java/io/delta/kernel/types/FieldMetadata.java index f2dc9edb0e3..e7a17a4654d 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/types/FieldMetadata.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/types/FieldMetadata.java @@ -172,17 +172,15 @@ public boolean equalsIgnoreKeys(FieldMetadata other, Set keys) { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - FieldMetadata that = (FieldMetadata) o; - if (this.metadata.size() != that.metadata.size()) return false; + FieldMetadata other = (FieldMetadata) o; + if (this.metadata.size() != other.metadata.size()) return false; return this.metadata.entrySet().stream() .allMatch( - e -> - Objects.equals(e.getValue(), that.metadata.get(e.getKey())) - || (e.getValue() != null - && e.getValue().getClass().isArray() - && that.metadata.get(e.getKey()).getClass().isArray() - && Arrays.equals( - (Object[]) e.getValue(), (Object[]) that.metadata.get(e.getKey())))); + e -> { + Object value = e.getValue(); + Object otherValue = other.metadata.get(e.getKey()); + return Objects.deepEquals(value, otherValue); + }); } @Override diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/types/MapType.java b/kernel/kernel-api/src/main/java/io/delta/kernel/types/MapType.java index b46fe327009..c1d4f2ab661 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/types/MapType.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/types/MapType.java @@ -33,15 +33,36 @@ public class MapType extends DataType { public static final String MAP_VALUE_NAME = "value"; public MapType(DataType keyType, DataType valueType, boolean valueContainsNull) { + validateKeyType(keyType); this.keyField = new StructField(MAP_KEY_NAME, keyType, false); this.valueField = new StructField(MAP_VALUE_NAME, valueType, valueContainsNull); } public MapType(StructField keyField, StructField valueField) { + validateKeyType(keyField.getDataType()); this.keyField = keyField; this.valueField = valueField; } + /** + * The Delta protocol does not support collated string types as map keys. Only StringType with the + * default UTF8_BINARY collation is allowed. + * + * @see + * Collated String Type RFC + */ + private static void validateKeyType(DataType keyType) { + if (keyType instanceof StringType && !((StringType) keyType).isUTF8BinaryCollated()) { + throw new IllegalArgumentException( + String.format( + "MapType does not support collated string types as keys. " + + "Found key type '%s', but only StringType with default UTF8_BINARY " + + "collation is allowed.", + keyType)); + } + } + public StructField getKeyField() { return keyField; } diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/TransactionBuilderImplSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/TransactionBuilderImplSuite.scala new file mode 100644 index 00000000000..8533cceafb7 --- /dev/null +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/TransactionBuilderImplSuite.scala @@ -0,0 +1,109 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.kernel.internal + +import java.util.Optional + +import scala.collection.JavaConverters._ + +import io.delta.kernel.Operation +import io.delta.kernel.internal.actions.{Format, Metadata, Protocol} +import io.delta.kernel.internal.checksum.CRCInfo +import io.delta.kernel.internal.commit.DefaultFileSystemManagedTableOnlyCommitter +import io.delta.kernel.internal.fs.Path +import io.delta.kernel.internal.lang.Lazy +import io.delta.kernel.internal.metrics.SnapshotQueryContext +import io.delta.kernel.internal.snapshot.LogSegment +import io.delta.kernel.internal.util.FileNames +import io.delta.kernel.internal.util.VectorUtils.{buildArrayValue, stringStringMapValue} +import io.delta.kernel.test.MockEngineUtils +import io.delta.kernel.types.{IntegerType, StringType, StructType} +import io.delta.kernel.utils.FileStatus + +import org.scalatest.funsuite.AnyFunSuite + +class TransactionBuilderImplSuite extends AnyFunSuite with MockEngineUtils { + + /** + * Creates a mock snapshot whose metadata includes a `delta.feature.` property that + * TransactionMetadataFactory would attempt to process. This is used to verify the early-return + * path: if the code falls through to the factory, processing the unknown feature throws. + */ + private def createMockSnapshot( + dataPath: Path, + version: Long, + extraConfig: Map[String, String] = Map.empty): SnapshotImpl = { + val schema = new StructType().add("id", IntegerType.INTEGER) + val metadata = new Metadata( + "id", + Optional.empty(), + Optional.empty(), + new Format(), + schema.toJson, + schema, + buildArrayValue(java.util.Arrays.asList(), StringType.STRING), + Optional.of(123), + stringStringMapValue(extraConfig.asJava)) + val logPath = new Path(dataPath, "_delta_log") + val fs = FileStatus.of(FileNames.deltaFile(logPath, version), 1, 1) + val logSegment = new LogSegment( + logPath, + version, + Seq(fs).asJava, + Seq.empty.asJava, + Seq.empty.asJava, + fs, + Optional.empty(), + Optional.empty()) + val snapshotQueryContext = SnapshotQueryContext.forLatestSnapshot(dataPath.toString) + new SnapshotImpl( + dataPath, + logSegment.getVersion, + new Lazy(() => logSegment), + null, // logReplay - not needed; getCurrentCrcInfo is overridden below + new Protocol(1, 2), + metadata, + DefaultFileSystemManagedTableOnlyCommitter.INSTANCE, + snapshotQueryContext, + Optional.empty()) { + override def getCurrentCrcInfo: Optional[CRCInfo] = Optional.empty() + } + } + + test("early return when no metadata or protocol update is needed") { + // The snapshot metadata includes an unrecognized delta.feature.* property. If the code + // falls through to TransactionMetadataFactory (the bug), extractFeaturePropertyOverrides + // will try to resolve this feature and throw. The early-return path skips the factory + // entirely, so no exception is thrown. + val dataPath = new Path("/tmp/test-table") + val tableImpl = new TableImpl(dataPath.toString, () => System.currentTimeMillis()) + val snapshot = createMockSnapshot( + dataPath, + version = 0L, + extraConfig = Map("delta.feature.fakeFeatureForEarlyReturnTest" -> "supported")) + + val builder = new TransactionBuilderImpl(tableImpl, "test-engine", Operation.WRITE) + + // With the fix this returns immediately; without the fix this would throw + val txn = builder.buildTransactionInternal( + mockEngine(), + false, // isCreateOrReplace + Optional.of(snapshot)) + + // Verify the returned transaction does not mark protocol/metadata for update + assert(txn.getSchema(mockEngine()) === snapshot.getMetadata().getSchema()) + } +} diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/actions/DeletionVectorDescriptorSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/actions/DeletionVectorDescriptorSuite.scala index aa7aabe256d..9fb3623c465 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/actions/DeletionVectorDescriptorSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/actions/DeletionVectorDescriptorSuite.scala @@ -21,7 +21,7 @@ import java.util.{Base64, Optional} import org.scalatest.funsuite.AnyFunSuite /** - * Tests for DeletionVectorDescriptor.serializeToBase64(). + * Tests for DeletionVectorDescriptor. */ class DeletionVectorDescriptorSuite extends AnyFunSuite { @@ -59,6 +59,21 @@ class DeletionVectorDescriptorSuite extends AnyFunSuite { } } + // Regression test: isInline() must use .equals() not == for String comparison. + // Using `new String(...)` creates non-interned Strings that would fail with ==. + testCases.foreach { case (storageType, pathOrInlineDv, offset, sizeInBytes, cardinality) => + test(s"isInline with non-interned string - $storageType storage type") { + val dv = new DeletionVectorDescriptor( + new String(storageType), // deliberately non-interned + pathOrInlineDv, + offset.map(Integer.valueOf).map(Optional.of[Integer]).getOrElse(Optional.empty[Integer]()), + sizeInBytes, + cardinality) + + assert(dv.isInline() === (storageType == "i")) + } + } + test("serializeToBase64 throws for non-inline DV without offset") { val ex = intercept[IllegalArgumentException] { val dv = new DeletionVectorDescriptor( diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/actions/ProtocolSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/actions/ProtocolSuite.scala index d90a5f9f62b..d96df5f2720 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/actions/ProtocolSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/actions/ProtocolSuite.scala @@ -290,6 +290,11 @@ class ProtocolSuite extends AnyFunSuite with TestUtils { 7, Set[String]("variantType", "variantShredding-preview"), Set[String]("variantType", "variantShredding-preview")), + Set("variantShredding") -> ( + 3, + 7, + Set[String]("variantType", "variantShredding"), + Set[String]("variantType", "variantShredding")), Set("rowTracking") -> ( 1, 7, diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/checksum/CRCInfoReadCompatSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/checksum/CRCInfoReadCompatSuite.scala new file mode 100644 index 00000000000..f07c6be77b7 --- /dev/null +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/checksum/CRCInfoReadCompatSuite.scala @@ -0,0 +1,219 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.kernel.internal.checksum + +import java.util +import java.util.{Collections, Optional} + +import scala.collection.JavaConverters._ + +import io.delta.kernel.data.{ColumnarBatch, ColumnVector, Row} +import io.delta.kernel.internal.actions.{DomainMetadata, Format, Metadata, Protocol} +import io.delta.kernel.internal.checksum.CRCInfo.{CRC_FILE_READ_SCHEMA, CRC_FILE_SCHEMA} +import io.delta.kernel.internal.data.GenericColumnVector +import io.delta.kernel.internal.stats.FileSizeHistogram +import io.delta.kernel.internal.types.DataTypeJsonSerDe +import io.delta.kernel.internal.util.VectorUtils +import io.delta.kernel.internal.util.VectorUtils.{buildArrayValue, stringStringMapValue} +import io.delta.kernel.test.VectorTestUtils +import io.delta.kernel.types.{DataType, StringType, StructType} + +import org.scalatest.funsuite.AnyFunSuite + +/** + * Tests that CRCInfo.fromColumnarBatch correctly reads the file size histogram from CRC files + * written using the legacy field name "histogramOpt" or the spec-compliant "fileSizeHistogram". + */ +class CRCInfoReadCompatSuite extends AnyFunSuite with VectorTestUtils { + + private val testProtocol = + new Protocol(1, 2, Collections.emptySet(), Collections.emptySet()) + + private val testMetadata = new Metadata( + "id", + Optional.of("name"), + Optional.of("description"), + new Format("parquet", Collections.emptyMap()), + DataTypeJsonSerDe.serializeDataType(new StructType()), + new StructType(), + buildArrayValue(util.Arrays.asList("c3"), StringType.STRING), + Optional.of(123), + stringStringMapValue(new util.HashMap[String, String]() { + put("delta.appendOnly", "true") + })) + + /** Creates a simple histogram with distinct values for identification in tests. */ + private def createTestHistogram(fileCount: Long): FileSizeHistogram = { + val boundaries = Array(0L, 1024L) + val counts = Array(fileCount, 0L) + val bytes = Array(fileCount * 100, 0L) + new FileSizeHistogram(boundaries, counts, bytes) + } + + /** + * Builds a ColumnVector for a FileSizeHistogram struct field. If histogram is None, the vector + * is null at row 0. + */ + private def histogramColumnVector( + histogram: Option[FileSizeHistogram]): ColumnVector = { + val rowValue: Row = histogram.map(_.toRow()).orNull + new GenericColumnVector( + util.Arrays.asList(rowValue), + FileSizeHistogram.FULL_SCHEMA) + } + + /** + * Build a ColumnarBatch with the given schema and histogram column vectors at the appropriate + * positions. + */ + private def buildBatch( + schema: StructType, + fileSizeHistogram: Option[FileSizeHistogram], + histogramOpt: Option[FileSizeHistogram]): ColumnarBatch = { + val protocolColVector = + new GenericColumnVector( + util.Arrays.asList(testProtocol.toRow()), + Protocol.FULL_SCHEMA) + val metadataColVector = + new GenericColumnVector( + util.Arrays.asList(testMetadata.toRow()), + Metadata.FULL_SCHEMA) + + new ColumnarBatch { + override def getSchema: StructType = schema + override def getSize: Int = 1 + + override def getColumnVector(ordinal: Int): ColumnVector = { + val fieldName = schema.at(ordinal).getName + fieldName match { + case "tableSizeBytes" => longVector(Seq(1000L)) + case "numFiles" => longVector(Seq(10L)) + case "numMetadata" => longVector(Seq(1L)) + case "numProtocol" => longVector(Seq(1L)) + case "metadata" => metadataColVector + case "protocol" => protocolColVector + case "txnId" => stringVector(Seq(null)) + case "domainMetadata" => nullColumnVector( + CRC_FILE_SCHEMA.get("domainMetadata").getDataType) + case "fileSizeHistogram" => histogramColumnVector(fileSizeHistogram) + case "histogramOpt" => histogramColumnVector(histogramOpt) + case _ => + throw new IllegalArgumentException(s"Unknown field: $fieldName") + } + } + } + } + + /** Creates a column vector that is null at every row. */ + private def nullColumnVector(dataType: DataType): ColumnVector = { + new ColumnVector { + override def getDataType: DataType = dataType + override def getSize: Int = 1 + override def close(): Unit = {} + override def isNullAt(rowId: Int): Boolean = true + } + } + + test("reads fileSizeHistogram when only spec-compliant field is present") { + val histogram = createTestHistogram(fileCount = 42) + val batch = buildBatch( + CRC_FILE_READ_SCHEMA, + fileSizeHistogram = Some(histogram), + histogramOpt = None) + + val crcInfo = CRCInfo.fromColumnarBatch(1L, batch, 0, "test.crc") + assert(crcInfo.isPresent) + assert(crcInfo.get().getFileSizeHistogram.isPresent) + assert(crcInfo.get().getFileSizeHistogram.get() === histogram) + } + + test("reads histogramOpt when only legacy field is present") { + val histogram = createTestHistogram(fileCount = 99) + val batch = buildBatch( + CRC_FILE_READ_SCHEMA, + fileSizeHistogram = None, + histogramOpt = Some(histogram)) + + val crcInfo = CRCInfo.fromColumnarBatch(1L, batch, 0, "test.crc") + assert(crcInfo.isPresent) + assert(crcInfo.get().getFileSizeHistogram.isPresent) + assert(crcInfo.get().getFileSizeHistogram.get() === histogram) + } + + test("prefers fileSizeHistogram when both fields are present") { + val specHistogram = createTestHistogram(fileCount = 10) + val legacyHistogram = createTestHistogram(fileCount = 20) + val batch = buildBatch( + CRC_FILE_READ_SCHEMA, + fileSizeHistogram = Some(specHistogram), + histogramOpt = Some(legacyHistogram)) + + val crcInfo = CRCInfo.fromColumnarBatch(1L, batch, 0, "test.crc") + assert(crcInfo.isPresent) + assert(crcInfo.get().getFileSizeHistogram.isPresent) + assert(crcInfo.get().getFileSizeHistogram.get() === specHistogram) + } + + test("returns empty histogram when neither field is present") { + val batch = buildBatch( + CRC_FILE_READ_SCHEMA, + fileSizeHistogram = None, + histogramOpt = None) + + val crcInfo = CRCInfo.fromColumnarBatch(1L, batch, 0, "test.crc") + assert(crcInfo.isPresent) + assert(!crcInfo.get().getFileSizeHistogram.isPresent) + } + + test("safely skips fallback when batch uses original CRC_FILE_SCHEMA") { + // When fromColumnarBatch is called with a batch using the original schema + // (without histogramOpt), the fallback should be safely skipped. + val protocolColVector = + new GenericColumnVector( + util.Arrays.asList(testProtocol.toRow()), + Protocol.FULL_SCHEMA) + val metadataColVector = + new GenericColumnVector( + util.Arrays.asList(testMetadata.toRow()), + Metadata.FULL_SCHEMA) + + val batch = new ColumnarBatch { + override def getSchema: StructType = CRC_FILE_SCHEMA + override def getSize: Int = 1 + override def getColumnVector(ordinal: Int): ColumnVector = { + val fieldName = CRC_FILE_SCHEMA.at(ordinal).getName + fieldName match { + case "tableSizeBytes" => longVector(Seq(1000L)) + case "numFiles" => longVector(Seq(10L)) + case "numMetadata" => longVector(Seq(1L)) + case "numProtocol" => longVector(Seq(1L)) + case "metadata" => metadataColVector + case "protocol" => protocolColVector + case "txnId" => stringVector(Seq(null)) + case "domainMetadata" => nullColumnVector( + CRC_FILE_SCHEMA.get("domainMetadata").getDataType) + case "fileSizeHistogram" => histogramColumnVector(None) + case _ => + throw new IllegalArgumentException(s"Unknown field: $fieldName") + } + } + } + + val crcInfo = CRCInfo.fromColumnarBatch(1L, batch, 0, "test.crc") + assert(crcInfo.isPresent) + assert(!crcInfo.get().getFileSizeHistogram.isPresent) + } +} diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergCompatV2MetadataValidatorAndUpdaterSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergCompatV2MetadataValidatorAndUpdaterSuite.scala index 482df6d5b77..8e2903fb95d 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergCompatV2MetadataValidatorAndUpdaterSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergCompatV2MetadataValidatorAndUpdaterSuite.scala @@ -23,7 +23,7 @@ import io.delta.kernel.exceptions.KernelException import io.delta.kernel.internal.actions.{Metadata, Protocol} import io.delta.kernel.internal.icebergcompat.IcebergCompatV2MetadataValidatorAndUpdater.validateAndUpdateIcebergCompatV2Metadata import io.delta.kernel.internal.tablefeatures.TableFeature -import io.delta.kernel.internal.tablefeatures.TableFeatures.{COLUMN_MAPPING_RW_FEATURE, ICEBERG_COMPAT_V2_W_FEATURE} +import io.delta.kernel.internal.tablefeatures.TableFeatures.{COLUMN_MAPPING_RW_FEATURE, DELETION_VECTORS_RW_FEATURE, ICEBERG_COMPAT_V2_W_FEATURE} import io.delta.kernel.test.TestFixtures import io.delta.kernel.types._ @@ -70,7 +70,7 @@ class IcebergCompatV2MetadataValidatorAndUpdaterSuite isNewTable: Boolean, metadata: Metadata, protocol: Protocol): Optional[Metadata] = { - validateAndUpdateIcebergCompatV2Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergCompatV2Metadata(isNewTable, metadata, protocol, Optional.empty()) } Seq(true, false).foreach { isNewTable => @@ -130,4 +130,39 @@ class IcebergCompatV2MetadataValidatorAndUpdaterSuite } } } + + /* --- prevProtocol DV check (concurrent writer protection) --- */ + + test("DV check catches deletion vectors enabled in previous protocol") { + val schema = new StructType().add("col", BooleanType.BOOLEAN) + val metadata = getCompatEnabledMetadata(schema) + // newProtocol does NOT support DVs + val newProtocol = getCompatEnabledProtocol() + // prevProtocol DOES support DVs (simulating a concurrent writer that enabled DVs) + val prevProtocol = Optional.of(getCompatEnabledProtocol(DELETION_VECTORS_RW_FEATURE)) + + val e = intercept[KernelException] { + validateAndUpdateIcebergCompatV2Metadata( + false, /* isCreatingNewTable */ + metadata, + newProtocol, + prevProtocol) + } + assert(e.getMessage.contains( + "Table features [deletionVectors] are incompatible with icebergCompatV2")) + } + + test("DV check passes when neither new nor previous protocol has DVs") { + val schema = new StructType().add("col", BooleanType.BOOLEAN) + val metadata = getCompatEnabledMetadata(schema) + val newProtocol = getCompatEnabledProtocol() + val prevProtocol = Optional.of(getCompatEnabledProtocol()) + + // Should not throw + validateAndUpdateIcebergCompatV2Metadata( + false, /* isCreatingNewTable */ + metadata, + newProtocol, + prevProtocol) + } } diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergCompatV3MetadataValidatorAndUpdateSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergCompatV3MetadataValidatorAndUpdateSuite.scala index 38677f50631..fd4a1c3c590 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergCompatV3MetadataValidatorAndUpdateSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergCompatV3MetadataValidatorAndUpdateSuite.scala @@ -80,7 +80,7 @@ class IcebergCompatV3MetadataValidatorAndUpdaterSuite isNewTable: Boolean, metadata: Metadata, protocol: Protocol): Optional[Metadata] = { - validateAndUpdateIcebergCompatV3Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergCompatV3Metadata(isNewTable, metadata, protocol, Optional.empty()) } Seq(true, false).foreach { isNewTable => @@ -90,7 +90,7 @@ class IcebergCompatV3MetadataValidatorAndUpdaterSuite val protocol = new Protocol(3, 7, Set.empty.asJava, Set("icebergCompatV3", "rowTracking").asJava) val e = intercept[KernelException] { - validateAndUpdateIcebergCompatV3Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergCompatV3Metadata(isNewTable, metadata, protocol, Optional.empty()) } assert(e.getMessage.contains( "icebergCompatV3: requires the feature 'columnMapping' to be enabled.")) @@ -126,13 +126,13 @@ class IcebergCompatV3MetadataValidatorAndUpdaterSuite if (isNewTable) { val updatedMetadata = - validateAndUpdateIcebergCompatV3Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergCompatV3Metadata(isNewTable, metadata, protocol, Optional.empty()) assert(updatedMetadata.isPresent) assert(updatedMetadata.get().getConfiguration.get("delta.columnMapping.mode") == "name") assert(TableConfig.ROW_TRACKING_ENABLED.fromMetadata(updatedMetadata.get())) } else { val e = intercept[KernelException] { - validateAndUpdateIcebergCompatV3Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergCompatV3Metadata(isNewTable, metadata, protocol, Optional.empty()) } assert(e.getMessage.contains( "The value 'none' for the property 'delta.columnMapping.mode' is" + diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite.scala index 490ac642fc4..4f449c47679 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite.scala @@ -24,9 +24,9 @@ import io.delta.kernel.internal.TableConfig import io.delta.kernel.internal.actions.{Metadata, Protocol} import io.delta.kernel.internal.icebergcompat.IcebergWriterCompatV1MetadataValidatorAndUpdater.validateAndUpdateIcebergWriterCompatV1Metadata import io.delta.kernel.internal.tablefeatures.TableFeature -import io.delta.kernel.internal.tablefeatures.TableFeatures.{COLUMN_MAPPING_RW_FEATURE, ICEBERG_COMPAT_V2_W_FEATURE, ICEBERG_WRITER_COMPAT_V1, TYPE_WIDENING_RW_FEATURE} +import io.delta.kernel.internal.tablefeatures.TableFeatures.{COLUMN_MAPPING_RW_FEATURE, DELETION_VECTORS_RW_FEATURE, ICEBERG_COMPAT_V2_W_FEATURE, ICEBERG_WRITER_COMPAT_V1, TYPE_WIDENING_RW_FEATURE} import io.delta.kernel.internal.util.ColumnMapping -import io.delta.kernel.types.{ByteType, DataType, DecimalType, FieldMetadata, IntegerType, LongType, ShortType, StructField, StructType, TypeChange} +import io.delta.kernel.types.{BooleanType, ByteType, DataType, DecimalType, FieldMetadata, IntegerType, LongType, ShortType, StructField, StructType, TypeChange} class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite extends IcebergCompatV2MetadataValidatorAndUpdaterSuiteBase { @@ -35,7 +35,7 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite isNewTable: Boolean, metadata: Metadata, protocol: Protocol): Optional[Metadata] = { - validateAndUpdateIcebergWriterCompatV1Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV1Metadata(isNewTable, metadata, protocol, Optional.empty()) } val icebergWriterCompatV1EnabledProps = Map( @@ -73,7 +73,8 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite validateAndUpdateIcebergWriterCompatV1Metadata( true, /* isNewTable */ metadata, - getCompatEnabledProtocol()) + getCompatEnabledProtocol(), + Optional.empty()) } /* --- UNSUPPORTED_TYPES_CHECK tests --- */ @@ -85,7 +86,11 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite val metadata = getCompatEnabledMetadata(schema) val protocol = getCompatEnabledProtocol() val e = intercept[KernelException] { - validateAndUpdateIcebergWriterCompatV1Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV1Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) } assert(e.getMessage.contains( s"icebergWriterCompatV1 does not support the data types: ")) @@ -113,7 +118,11 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite if (isNewTable) { val updatedMetadata = - validateAndUpdateIcebergWriterCompatV1Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV1Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) assert(updatedMetadata.isPresent) assert(updatedMetadata.get().getConfiguration.get("delta.columnMapping.mode") == "id") // We correctly populate the column mapping metadata @@ -124,7 +133,11 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite enableIcebergWriterCompatV1 = true) } else { val e = intercept[KernelException] { - validateAndUpdateIcebergWriterCompatV1Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV1Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) } assert(e.getMessage.contains( "The value 'none' for the property 'delta.columnMapping.mode' is" + @@ -145,7 +158,11 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite val protocol = getCompatEnabledProtocol() val e = intercept[KernelException] { - validateAndUpdateIcebergWriterCompatV1Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV1Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) } assert(e.getMessage.contains( s"The value '$cmMode' for the property 'delta.columnMapping.mode' is" + @@ -169,7 +186,11 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite val protocol = getCompatEnabledProtocol() val e = intercept[KernelException] { - validateAndUpdateIcebergWriterCompatV1Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV1Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) } assert(e.getMessage.contains( "IcebergWriterCompatV1 requires column mapping field physical names be equal to " @@ -194,7 +215,11 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite val protocol = getCompatEnabledProtocol() val updatedMetadata = - validateAndUpdateIcebergWriterCompatV1Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV1Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) // No metadata update happens assert(!updatedMetadata.isPresent) } @@ -213,12 +238,20 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite if (isNewTable) { val updatedMetadata = - validateAndUpdateIcebergWriterCompatV1Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV1Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) assert(updatedMetadata.isPresent) assert(TableConfig.ICEBERG_COMPAT_V2_ENABLED.fromMetadata(updatedMetadata.get)) } else { val e = intercept[KernelException] { - validateAndUpdateIcebergWriterCompatV1Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV1Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) } assert(e.getMessage.contains( "The value 'false' for the property 'delta.enableIcebergCompatV2' is" + @@ -237,7 +270,11 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite val protocol = getCompatEnabledProtocol() val e = intercept[KernelException] { - validateAndUpdateIcebergWriterCompatV1Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV1Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) } assert(e.getMessage.contains( "The value 'false' for the property 'delta.enableIcebergCompatV2' is" + @@ -275,8 +312,8 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite "catalogManaged") val protocol = new Protocol(3, 7, readerFeatures.asJava, writerFeatures.asJava) val metadata = getCompatEnabledMetadata(cmTestSchema()) - validateAndUpdateIcebergWriterCompatV1Metadata(true, metadata, protocol) - validateAndUpdateIcebergWriterCompatV1Metadata(false, metadata, protocol) + validateAndUpdateIcebergWriterCompatV1Metadata(true, metadata, protocol, Optional.empty()) + validateAndUpdateIcebergWriterCompatV1Metadata(false, metadata, protocol, Optional.empty()) } test("compatible type widening is allowed with icebergWriterCompatV1") { @@ -298,7 +335,7 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite val protocol = getCompatEnabledProtocol(TYPE_WIDENING_RW_FEATURE) // This should not throw an exception - validateAndUpdateIcebergWriterCompatV1Metadata(false, metadata, protocol) + validateAndUpdateIcebergWriterCompatV1Metadata(false, metadata, protocol, Optional.empty()) } test("incompatible type widening throws exception with icebergWriterCompatV1") { @@ -319,7 +356,7 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite val protocol = getCompatEnabledProtocol(TYPE_WIDENING_RW_FEATURE) val e = intercept[KernelException] { - validateAndUpdateIcebergWriterCompatV1Metadata(false, metadata, protocol) + validateAndUpdateIcebergWriterCompatV1Metadata(false, metadata, protocol, Optional.empty()) } assert(e.getMessage.contains("icebergCompatV2 does not support type widening present in table")) @@ -340,7 +377,11 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite val metadata = getCompatEnabledMetadata(cmTestSchema()) Seq(true, false).foreach { isNewTable => val e = intercept[KernelException] { - validateAndUpdateIcebergWriterCompatV1Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV1Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) } assert(e.getMessage.contains(expectedErrorMessageContains)) } @@ -348,7 +389,8 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite Seq( // "defaultColumns", add this to this test once we support defaultColumns - // "collations", add this to this test once we support collations + "collations", + "collations-preview", "variantType").foreach { incompatibleFeature => test(s"cannot enable with incompatible feature $incompatibleFeature") { checkUnsupportedOrIncompatibleFeature( @@ -358,7 +400,7 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite } } - Seq("collations", "defaultColumns").foreach { unsupportedIncompatibleFeature => + Seq("defaultColumns").foreach { unsupportedIncompatibleFeature => test(s"cannot enable with incompatible UNSUPPORTED feature $unsupportedIncompatibleFeature") { // We add this test here so that it will fail when we add Kernel support for these features // When this happens -> add the feature to the test above @@ -379,7 +421,8 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite validateAndUpdateIcebergWriterCompatV1Metadata( isNewTable, activeFeatureMetadata, - getCompatEnabledProtocol()) + getCompatEnabledProtocol(), + Optional.empty()) } assert(e.getMessage.contains( s"Table features [$tableFeature] are incompatible with icebergWriterCompatV1")) @@ -457,7 +500,11 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite val writerFeatures = Set("icebergWriterCompatV1", featureToIncludeStr) val protocol = new Protocol(3, 7, readerFeatures.asJava, writerFeatures.asJava) val e = intercept[KernelException] { - validateAndUpdateIcebergWriterCompatV1Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV1Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) } // Since we run icebergCompatV2 validation as part of // ICEBERG_COMPAT_V2_ENABLED.postProcess we actually hit the missing feature error in the @@ -467,4 +514,25 @@ class IcebergWriterCompatV1MetadataValidatorAndUpdaterSuite } } } + + /* --- prevProtocol DV check flows through WriterCompatV1 -> CompatV2 delegation --- */ + + test("prevProtocol DV check flows through WriterCompatV1 delegation to CompatV2") { + val schema = new StructType().add("col", BooleanType.BOOLEAN) + val metadata = getCompatEnabledMetadata(schema) + val newProtocol = getCompatEnabledProtocol() + // prevProtocol has DVs enabled, simulating a table that had DVs before this transaction + val prevProtocol = Optional.of(getCompatEnabledProtocol(DELETION_VECTORS_RW_FEATURE)) + + val e = intercept[KernelException] { + validateAndUpdateIcebergWriterCompatV1Metadata( + false, + metadata, + newProtocol, + prevProtocol) + } + // The error comes from icebergCompatV2 since WriterCompatV1 delegates DV checking to V2 + assert(e.getMessage.contains( + "Table features [deletionVectors] are incompatible with icebergCompatV2")) + } } diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite.scala index 25965600827..05e09e34e59 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/icebergcompat/IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite.scala @@ -34,7 +34,7 @@ class IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite isNewTable: Boolean, metadata: Metadata, protocol: Protocol): Optional[Metadata] = { - validateAndUpdateIcebergWriterCompatV3Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV3Metadata(isNewTable, metadata, protocol, Optional.empty()) } val icebergWriterCompatV3EnabledProps = Map( @@ -76,6 +76,7 @@ class IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite ICEBERG_COMPAT_V3_W_FEATURE, DELETION_VECTORS_RW_FEATURE, VARIANT_RW_FEATURE, + VARIANT_SHREDDING_RW_FEATURE, VARIANT_SHREDDING_PREVIEW_RW_FEATURE, VARIANT_RW_PREVIEW_FEATURE, ROW_TRACKING_W_FEATURE, @@ -100,12 +101,17 @@ class IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite "rowTracking", "variantType", "variantType-preview", + "variantShredding", "variantShredding-preview", tableFeature).asJava) val metadata = getCompatEnabledMetadata(cmTestSchema()) Seq(true, false).foreach { isNewTable => val e = intercept[KernelException] { - validateAndUpdateIcebergWriterCompatV3Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV3Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) } assert(e.getMessage.contains(expectedErrorMessageContains)) } @@ -120,7 +126,8 @@ class IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite validateAndUpdateIcebergWriterCompatV3Metadata( isNewTable, activeFeatureMetadata, - getCompatEnabledProtocol()) + getCompatEnabledProtocol(), + Optional.empty()) } assert(e.getMessage.contains( s"Table features [$tableFeature] are incompatible with icebergWriterCompatV3")) @@ -148,7 +155,11 @@ class IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite if (isNewTable) { val updatedMetadata = - validateAndUpdateIcebergWriterCompatV3Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV3Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) assert(updatedMetadata.isPresent) assert(updatedMetadata.get().getConfiguration.get("delta.columnMapping.mode") == "id") // We correctly populate the column mapping metadata @@ -159,7 +170,11 @@ class IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite true) } else { val e = intercept[KernelException] { - validateAndUpdateIcebergWriterCompatV3Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV3Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) } assert(e.getMessage.contains( "The value 'none' for the property 'delta.columnMapping.mode' is" + @@ -177,7 +192,8 @@ class IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite validateAndUpdateIcebergWriterCompatV3Metadata( true, /* isNewTable */ metadata, - getCompatEnabledProtocol()) + getCompatEnabledProtocol(), + Optional.empty()) } Seq("name", "none").foreach { cmMode => @@ -191,7 +207,11 @@ class IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite val protocol = getCompatEnabledProtocol() val e = intercept[KernelException] { - validateAndUpdateIcebergWriterCompatV3Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV3Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) } assert(e.getMessage.contains( s"The value '$cmMode' for the property 'delta.columnMapping.mode' is" + @@ -215,7 +235,11 @@ class IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite val protocol = getCompatEnabledProtocol() val e = intercept[KernelException] { - validateAndUpdateIcebergWriterCompatV3Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV3Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) } assert(e.getMessage.contains( "requires column mapping field physical names be equal to " @@ -240,7 +264,11 @@ class IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite val protocol = getCompatEnabledProtocol() val updatedMetadata = - validateAndUpdateIcebergWriterCompatV3Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV3Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) // No metadata update happens assert(!updatedMetadata.isPresent) } @@ -255,7 +283,11 @@ class IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite val metadata = getCompatEnabledMetadata(schema) val protocol = getCompatEnabledProtocol() val e = intercept[KernelException] { - validateAndUpdateIcebergWriterCompatV3Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV3Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) } assert(e.getMessage.contains( s"icebergWriterCompatV3 does not support the data types: ")) @@ -321,12 +353,20 @@ class IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite if (isNewTable) { val updatedMetadata = - validateAndUpdateIcebergWriterCompatV3Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV3Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) assert(updatedMetadata.isPresent) assert(TableConfig.ICEBERG_COMPAT_V3_ENABLED.fromMetadata(updatedMetadata.get)) } else { val e = intercept[KernelException] { - validateAndUpdateIcebergWriterCompatV3Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV3Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) } assert(e.getMessage.contains( "The value 'false' for the property 'delta.enableIcebergCompatV3' is" + @@ -346,7 +386,11 @@ class IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite val protocol = getCompatEnabledProtocol() val e = intercept[KernelException] { - validateAndUpdateIcebergWriterCompatV3Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV3Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) } assert(e.getMessage.contains( "The value 'false' for the property 'delta.enableIcebergCompatV3' is" + @@ -383,17 +427,18 @@ class IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite "rowTracking", "variantType", "variantType-preview", + "variantShredding", "variantShredding-preview", "icebergCompatV2", "icebergWriterCompatV1", "catalogManaged") val protocol = new Protocol(3, 7, readerFeatures.asJava, writerFeatures.asJava) val metadata = getCompatEnabledMetadata(cmTestSchema()) - validateAndUpdateIcebergWriterCompatV3Metadata(true, metadata, protocol) - validateAndUpdateIcebergWriterCompatV3Metadata(false, metadata, protocol) + validateAndUpdateIcebergWriterCompatV3Metadata(true, metadata, protocol, Optional.empty()) + validateAndUpdateIcebergWriterCompatV3Metadata(false, metadata, protocol, Optional.empty()) } - Seq("collations", "defaultColumns").foreach { unsupportedIncompatibleFeature => + Seq("defaultColumns").foreach { unsupportedIncompatibleFeature => test(s"cannot enable with incompatible UNSUPPORTED feature $unsupportedIncompatibleFeature") { // We add this test here so that it will fail when we add Kernel support for these features // When this happens -> add the feature to the test above @@ -475,7 +520,11 @@ class IcebergWriterCompatV3MetadataValidatorAndUpdaterSuite val writerFeatures = Set("icebergWriterCompatV3", featureToIncludeStr) val protocol = new Protocol(3, 7, readerFeatures.asJava, writerFeatures.asJava) val e = intercept[KernelException] { - validateAndUpdateIcebergWriterCompatV3Metadata(isNewTable, metadata, protocol) + validateAndUpdateIcebergWriterCompatV3Metadata( + isNewTable, + metadata, + protocol, + Optional.empty()) } // Since we run icebergCompatV3 validation as part of // ICEBERG_COMPAT_V3_ENABLED.postProcess we actually hit the missing feature error in the diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/tablefeatures/TableFeaturesSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/tablefeatures/TableFeaturesSuite.scala index 6da7863e6ae..1360d55472f 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/tablefeatures/TableFeaturesSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/tablefeatures/TableFeaturesSuite.scala @@ -50,7 +50,9 @@ class TableFeaturesSuite extends AnyFunSuite { "vacuumProtocolCheck", "variantType", "variantType-preview", - "variantShredding-preview") + "variantShredding", + "variantShredding-preview", + "geospatial") val writerOnlyFeatures = Seq( "allowColumnDefaults", @@ -69,7 +71,9 @@ class TableFeaturesSuite extends AnyFunSuite { "icebergWriterCompatV1", "icebergWriterCompatV3", "clustering", - "materializePartitionColumns") + "materializePartitionColumns", + "collations", + "collations-preview") val legacyFeatures = Seq( "appendOnly", @@ -132,11 +136,11 @@ class TableFeaturesSuite extends AnyFunSuite { ("rowTracking", testMetadata(tblProps = Map("delta.enableRowTracking" -> "true")), true), ("rowTracking", testMetadata(tblProps = Map("delta.enableRowTracking" -> "false")), false), ( - "variantShredding-preview", + "variantShredding", testMetadata(tblProps = Map("delta.enableVariantShredding" -> "true")), true), ( - "variantShredding-preview", + "variantShredding", testMetadata(tblProps = Map("delta.enableVariantShredding" -> "false")), false), ( @@ -149,6 +153,8 @@ class TableFeaturesSuite extends AnyFunSuite { false), ("timestampNtz", testMetadata(includeTimestampNtzTypeCol = true), true), ("timestampNtz", testMetadata(includeTimestampNtzTypeCol = false), false), + ("collations", testMetadata(includeCollatedStringTypeCol = true), true), + ("collations", testMetadata(includeCollatedStringTypeCol = false), false), ("v2Checkpoint", testMetadata(tblProps = Map("delta.checkpointPolicy" -> "v2")), true), ("v2Checkpoint", testMetadata(tblProps = Map("delta.checkpointPolicy" -> "classic")), false), ( @@ -190,7 +196,10 @@ class TableFeaturesSuite extends AnyFunSuite { ( "icebergWriterCompatV3", testMetadata(tblProps = Map("delta.enableIcebergWriterCompatV3" -> "false")), - false)).foreach({ case (feature, metadata, expected) => + false), + ("geospatial", testMetadata(includeGeometryTypeCol = true), true), + ("geospatial", testMetadata(includeGeographyTypeCol = true), true), + ("geospatial", testMetadata(), false)).foreach({ case (feature, metadata, expected) => test(s"metadataRequiresFeatureToBeEnabled - $feature - $metadata") { val tableFeature = TableFeatures.getTableFeature(feature) assert(tableFeature.isInstanceOf[FeatureAutoEnabledByMetadata]) @@ -205,7 +214,9 @@ class TableFeaturesSuite extends AnyFunSuite { "vacuumProtocolCheck", "clustering", "catalogManaged", - "allowColumnDefaults").foreach { + "allowColumnDefaults", + "variantShredding-preview", + "collations-preview").foreach { feature => test(s"doesn't support auto enable by metadata: $feature") { val tableFeature = TableFeatures.getTableFeature(feature) @@ -215,7 +226,11 @@ class TableFeaturesSuite extends AnyFunSuite { Seq( ("variantType", testMetadata(includeVariantTypeCol = true)), - ("typeWidening", testMetadata(tblProps = Map("delta.enableTypeWidening" -> "true")))).foreach { + ("typeWidening", testMetadata(tblProps = Map("delta.enableTypeWidening" -> "true"))), + ( + "variantShredding", + testMetadata(tblProps = Map("delta.enableVariantShredding" -> "true"))), + ("collations", testMetadata(includeCollatedStringTypeCol = true))).foreach { case (feature, metadataEnablingFeature) => test("special handling of tables containing preview features: " + feature) { val protocolWithPreviewFeature = new Protocol(3, 7) @@ -242,12 +257,14 @@ class TableFeaturesSuite extends AnyFunSuite { "v2Checkpoint", "variantType", "variantType-preview", + "variantShredding", "variantShredding-preview", "typeWidening", "typeWidening-preview", "deletionVectors", "timestampNtz", - "vacuumProtocolCheck") + "vacuumProtocolCheck", + "geospatial") assert(results.map(_.featureName()).toSet == expected.toSet) } @@ -287,8 +304,12 @@ class TableFeaturesSuite extends AnyFunSuite { "clustering", "variantType-preview", "variantType", + "variantShredding", "variantShredding-preview", - "materializePartitionColumns") + "materializePartitionColumns", + "collations", + "collations-preview", + "geospatial") assert(results.map(_.featureName()).toSet == expected.toSet) } @@ -331,6 +352,7 @@ class TableFeaturesSuite extends AnyFunSuite { "catalogManaged", "variantType", "variantType-preview", + "variantShredding", "variantShredding-preview", "deletionVectors", "typeWidening", @@ -339,7 +361,8 @@ class TableFeaturesSuite extends AnyFunSuite { "v2Checkpoint", "vacuumProtocolCheck", "allowColumnDefaults", - "columnMapping").foreach { feature => + "columnMapping", + "geospatial").foreach { feature => test(s"validateKernelCanReadTheTable: protocol 3 with $feature") { val protocol = new Protocol(3, 1, singleton(feature), Set().asJava) validateKernelCanReadTheTable(protocol, "/test/table") @@ -440,7 +463,8 @@ class TableFeaturesSuite extends AnyFunSuite { new Protocol(1, 4), testMetadata(includeGeneratedColumn = true)) - checkWriteUnsupported( + // Supported if append-only or remove-only + checkWriteSupported( "validateKernelCanWriteToTable: protocol 4 with changeDataFeed", new Protocol(1, 4), testMetadata(tblProps = Map("delta.enableChangeDataFeed" -> "true"))) @@ -507,7 +531,8 @@ class TableFeaturesSuite extends AnyFunSuite { new Protocol(1, 7, Set().asJava, singleton("changeDataFeed")), testMetadata()) - checkWriteUnsupported( + // Supported if append-only or remove-only + checkWriteSupported( "validateKernelCanWriteToTable: protocol 7 with changeDataFeed, " + "metadata contains changeDataFeed", new Protocol(1, 7, Set().asJava, singleton("changeDataFeed")), @@ -581,19 +606,20 @@ class TableFeaturesSuite extends AnyFunSuite { testMetadata(tblProps = Map("delta.enableTypeWidening" -> "true"))) } - Seq("variantType", "variantType-preview", "variantShredding-preview").foreach { feature => - checkWriteSupported( - s"validateKernelCanWriteToTable: protocol 7 with $feature, " + - s"metadata doesn't contains $feature", - new Protocol(3, 7, singleton(feature), singleton(feature)), - testMetadata()) - - checkWriteSupported( - s"validateKernelCanWriteToTable: protocol 7 with $feature, " + - s"metadata contains $feature", - new Protocol(3, 7, singleton(feature), singleton(feature)), - testMetadata(includeVariantTypeCol = true)) - } + Seq("variantType", "variantType-preview", "variantShredding", "variantShredding-preview") + .foreach { feature => + checkWriteSupported( + s"validateKernelCanWriteToTable: protocol 7 with $feature, " + + s"metadata doesn't contains $feature", + new Protocol(3, 7, singleton(feature), singleton(feature)), + testMetadata()) + + checkWriteSupported( + s"validateKernelCanWriteToTable: protocol 7 with $feature, " + + s"metadata contains $feature", + new Protocol(3, 7, singleton(feature), singleton(feature)), + testMetadata(includeVariantTypeCol = true)) + } checkWriteSupported( "validateKernelCanWriteToTable: protocol 7 with vacuumProtocolCheck, " + @@ -652,6 +678,18 @@ class TableFeaturesSuite extends AnyFunSuite { new Protocol(3, 7, emptySet(), singleton("materializePartitionColumns")), testMetadata()) + checkWriteSupported( + "validateKernelCanWriteToTable: protocol 7 with geospatial, " + + "metadata doesn't contain geospatial", + new Protocol(3, 7, singleton("geospatial"), singleton("geospatial")), + testMetadata()) + + checkWriteSupported( + "validateKernelCanWriteToTable: protocol 7 with geospatial, " + + "metadata contains geospatial", + new Protocol(3, 7, singleton("geospatial"), singleton("geospatial")), + testMetadata(includeGeometryTypeCol = true)) + checkWriteSupported( "validateKernelCanWriteToTable: protocol 7 with multiple features supported", new Protocol( @@ -1043,9 +1081,27 @@ class TableFeaturesSuite extends AnyFunSuite { set( "columnMapping", "deletionVectors", - "variantShredding-preview", + "variantShredding", "variantType")), - set("variantType", "variantShredding-preview"))).foreach { + set("variantType", "variantShredding")), + ( + testMetadata(includeGeometryTypeCol = true), + new Protocol(1, 1), + new Protocol( + 3, + 7, + set("geospatial"), + set("geospatial")), + set("geospatial")), + ( + testMetadata(includeGeographyTypeCol = true), + new Protocol(1, 2), + new Protocol( + 3, + 7, + set("geospatial"), + set("geospatial", "appendOnly", "invariants")), + set("geospatial"))).foreach { case (newMetadata, currentProtocol, expectedProtocol, expectedNewFeatures) => test(s"autoUpgradeProtocolBasedOnMetadata:" + s"$currentProtocol -> $expectedProtocol, $expectedNewFeatures") { @@ -1145,7 +1201,18 @@ class TableFeaturesSuite extends AnyFunSuite { 3, 7, set("columnMapping", "deletionVectors"), - set("columnMapping", "deletionVectors", "icebergCompatV2")))).foreach { + set("columnMapping", "deletionVectors", "icebergCompatV2"))), + ( + // Enable the variantShredding GA feature when the preview feature is already enabled. + // The GA feature should not be auto-enabled. + testMetadata( + tblProps = Map("delta.enableVariantShredding" -> "true"), + includeVariantTypeCol = true), + new Protocol( + 3, + 7, + set("variantType", "variantShredding-preview"), + set("variantType", "variantShredding-preview")))).foreach { case (newMetadata, currentProtocol) => test(s"autoUpgradeProtocolBasedOnMetadata: no-op upgrade: $currentProtocol") { val newProtocolAndNewFeaturesEnabled = @@ -1236,13 +1303,19 @@ class TableFeaturesSuite extends AnyFunSuite { includeVariantTypeCol: Boolean = false, includeGeneratedColumn: Boolean = false, includeIdentityColumn: Boolean = false, + includeGeometryTypeCol: Boolean = false, + includeGeographyTypeCol: Boolean = false, + includeCollatedStringTypeCol: Boolean = false, tblProps: Map[String, String] = Map.empty): Metadata = { val testSchema = createTestSchema( includeInvariant, includeTimestampNtzTypeCol, includeVariantTypeCol, includeGeneratedColumn, - includeIdentityColumn) + includeIdentityColumn, + includeGeometryTypeCol, + includeGeographyTypeCol, + includeCollatedStringTypeCol) new Metadata( "id", Optional.of("name"), @@ -1270,7 +1343,10 @@ class TableFeaturesSuite extends AnyFunSuite { includeTimestampNtzTypeCol: Boolean = false, includeVariantTypeCol: Boolean = false, includeGeneratedColumn: Boolean = false, - includeIdentityColumn: Boolean = false): StructType = { + includeIdentityColumn: Boolean = false, + includeGeometryTypeCol: Boolean = false, + includeGeographyTypeCol: Boolean = false, + includeCollatedStringTypeCol: Boolean = false): StructType = { var structType = new StructType() .add("c1", IntegerType.INTEGER) .add("c2", StringType.STRING) @@ -1306,6 +1382,15 @@ class TableFeaturesSuite extends AnyFunSuite { .putBoolean("delta.identity.allowExplicitInsert", true) .build()) } + if (includeGeometryTypeCol) { + structType = structType.add("c8", GeometryType.ofDefault()) + } + if (includeGeographyTypeCol) { + structType = structType.add("c9", GeographyType.ofDefault()) + } + if (includeCollatedStringTypeCol) { + structType = structType.add("c8", new StringType("ICU.UNICODE.75")) + } structType } diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/types/DataTypeJsonSerDeSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/types/DataTypeJsonSerDeSuite.scala index 700774bda1a..972f22bd8f7 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/types/DataTypeJsonSerDeSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/types/DataTypeJsonSerDeSuite.scala @@ -253,6 +253,22 @@ class DataTypeJsonSerDeSuite extends AnyFunSuite { } } + test("deserialize: schema with collated map key throws IllegalArgumentException") { + // A JSON schema encoding a MapType with a collated StringType key should fail + // deserialization because MapType rejects collated keys. + val json = structTypeJson(Seq( + structFieldJson( + "m", + mapTypeJson("\"string\"", "\"integer\"", false), + true, + metadataJson = Some( + s"""{"$COLLATIONS_METADATA_KEY" : {"m.key" : "ICU.UNICODE_CI"}}""")))) + + intercept[IllegalArgumentException] { + parse(json) + } + } + test("serialize/deserialize: parsed and original struct" + " type differing just in StringType collation") { val json = structTypeJson(Seq( @@ -370,7 +386,6 @@ class DataTypeJsonSerDeSuite extends AnyFunSuite { metadataJson = Some( s"""{ |"$COLLATIONS_METADATA_KEY": { - | "tags.key": "SPARK.UTF8_LCASE", | "tags.value": "ICU.UNICODE" |}, |"delta.typeChanges": [ @@ -386,7 +401,7 @@ class DataTypeJsonSerDeSuite extends AnyFunSuite { .add( "tags", new MapType( - new StructField("key", new StringType("SPARK.UTF8_LCASE"), false), + new StructField("key", StringType.STRING, false), new StructField("value", new StringType("ICU.UNICODE"), false) .withTypeChanges(Seq(new TypeChange(BinaryType.BINARY, StringType.STRING)).asJava)), true) @@ -745,8 +760,7 @@ object DataTypeJsonSerDeSuite { false, metadataJson = Some( s"""{"$COLLATIONS_METADATA_KEY" - | : {"b2.key" : "ICU.UNICODE_CI", - | "b2.value" : "SPARK.UTF8_LCASE"}}""".stripMargin)), + | : {"b2.value" : "SPARK.UTF8_LCASE"}}""".stripMargin)), structFieldJson("b3", arrayTypeJson("\"string\"", false), true), structFieldJson("b4", mapTypeJson("\"string\"", "\"string\"", false), false))), true), @@ -772,7 +786,7 @@ object DataTypeJsonSerDeSuite { .add( "b2", new MapType( - new StringType("ICU.UNICODE_CI"), + StringType.STRING, new StringType("SPARK.UTF8_LCASE"), true), false) @@ -839,10 +853,7 @@ object DataTypeJsonSerDeSuite { | : {"b1" : "SPARK.UTF8_LCASE"}}""".stripMargin)))), false), false), - true, - metadataJson = Some( - s"""{"$COLLATIONS_METADATA_KEY" - | : {"a3.element.key" : "ICU.UNICODE_CI"}}""".stripMargin)), + true), structFieldJson( "a4", arrayTypeJson( @@ -894,7 +905,7 @@ object DataTypeJsonSerDeSuite { "a3", new ArrayType( new MapType( - new StringType("ICU.UNICODE_CI"), + StringType.STRING, new StructType() .add("b1", new StringType("SPARK.UTF8_LCASE"), false), false), diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/util/DataFileStatisticsSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/util/DataFileStatisticsSuite.scala index 088929b6029..91bac545bd6 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/util/DataFileStatisticsSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/util/DataFileStatisticsSuite.scala @@ -1027,4 +1027,500 @@ class DataFileStatisticsSuite extends AnyFunSuite with Matchers { assert(exception.getMessage === expectedMessage) } + test("serialization with geometry and geography types") { + val schema = new StructType() + .add("geom", GeometryType.ofDefault()) + .add("geog", GeographyType.ofDefault()) + + val minValues = Map( + new Column("geom") -> Literal.ofGeospatialWKT( + "POINT (1 2)", + GeometryType.ofDefault()), + new Column("geog") -> Literal.ofGeospatialWKT( + "POINT (10 20)", + GeographyType.ofDefault())).asJava + + val maxValues = Map( + new Column("geom") -> Literal.ofGeospatialWKT( + "POINT (3 4)", + GeometryType.ofDefault()), + new Column("geog") -> Literal.ofGeospatialWKT( + "POINT (30 40)", + GeographyType.ofDefault())).asJava + + val nullCount = Map( + new Column("geom") -> java.lang.Long.valueOf(1L), + new Column("geog") -> java.lang.Long.valueOf(2L)).asJava + + val stats = new DataFileStatistics( + 100, + minValues, + maxValues, + nullCount, + Optional.of(true)) + + val json = stats.serializeAsJson(schema) + val expectedJson = + """{ + | "numRecords": 100, + | "minValues": { + | "geom": "POINT (1 2)", + | "geog": "POINT (10 20)" + | }, + | "maxValues": { + | "geom": "POINT (3 4)", + | "geog": "POINT (30 40)" + | }, + | "nullCount": { + | "geom": 1, + | "geog": 2 + | }, + | "tightBounds": true + |}""".stripMargin + + assert(areJsonNodesEqual(json, expectedJson)) + } + + test("deserialization with geometry and geography types") { + val schema = new StructType() + .add("geom", GeometryType.ofDefault()) + .add("geog", GeographyType.ofDefault()) + + val json = + """{ + | "numRecords": 50, + | "minValues": { + | "geom": "POINT (1 2)", + | "geog": "POINT (10 20)" + | }, + | "maxValues": { + | "geom": "POINT (3 4)", + | "geog": "POINT (30 40)" + | }, + | "nullCount": { + | "geom": 3, + | "geog": 0 + | }, + | "tightBounds": false + |}""".stripMargin + + val result = DataFileStatistics.deserializeFromJson(json, schema) + assert(result.isPresent) + + val stats = result.get() + assert(stats.getNumRecords == 50) + + val minValues = stats.getMinValues + assert(minValues.get(new Column("geom")).getValue == "POINT (1 2)") + assert(minValues.get(new Column("geom")).getDataType.isInstanceOf[GeometryType]) + assert(minValues.get(new Column("geog")).getValue == "POINT (10 20)") + assert(minValues.get(new Column("geog")).getDataType.isInstanceOf[GeographyType]) + + val maxValues = stats.getMaxValues + assert(maxValues.get(new Column("geom")).getValue == "POINT (3 4)") + assert(maxValues.get(new Column("geog")).getValue == "POINT (30 40)") + + assert(stats.getNullCount.get(new Column("geom")) == 3L) + assert(stats.getNullCount.get(new Column("geog")) == 0L) + assert(stats.getTightBounds.isPresent && !stats.getTightBounds.get) + } + + test("round-trip serialization for geometry and geography types") { + val schema = new StructType() + .add("geom", GeometryType.ofDefault()) + .add("geog", GeographyType.ofDefault()) + + val minValues = Map( + new Column("geom") -> Literal.ofGeospatialWKT( + "POINT (1.5 2.5)", + GeometryType.ofDefault()), + new Column("geog") -> Literal.ofGeospatialWKT( + "POINT (10.5 20.5)", + GeographyType.ofDefault())).asJava + + val maxValues = Map( + new Column("geom") -> Literal.ofGeospatialWKT( + "POINT (100 200)", + GeometryType.ofDefault()), + new Column("geog") -> Literal.ofGeospatialWKT( + "POINT (50 60)", + GeographyType.ofDefault())).asJava + + val nullCount = Map( + new Column("geom") -> java.lang.Long.valueOf(0L), + new Column("geog") -> java.lang.Long.valueOf(5L)).asJava + + val originalStats = new DataFileStatistics( + 200, + minValues, + maxValues, + nullCount, + Optional.of(true)) + + val json = originalStats.serializeAsJson(schema) + val deserialized = DataFileStatistics.deserializeFromJson(json, schema) + assert(deserialized.isPresent) + + val stats = deserialized.get() + assert(stats.getNumRecords == 200) + assert(stats.getMinValues.get(new Column("geom")).getValue == + "POINT (1.5 2.5)") + assert(stats.getMinValues.get(new Column("geom")) + .getDataType.isInstanceOf[GeometryType]) + assert(stats.getMinValues.get(new Column("geog")).getValue == + "POINT (10.5 20.5)") + assert(stats.getMinValues.get(new Column("geog")) + .getDataType.isInstanceOf[GeographyType]) + assert(stats.getMaxValues.get(new Column("geom")).getValue == + "POINT (100 200)") + assert(stats.getMaxValues.get(new Column("geog")).getValue == + "POINT (50 60)") + assert(stats.getNullCount.get(new Column("geom")) == 0L) + assert(stats.getNullCount.get(new Column("geog")) == 5L) + } + + test("serialization with null geospatial stats literal") { + val schema = new StructType() + .add("geom", GeometryType.ofDefault()) + .add("geog", GeographyType.ofDefault()) + + val minValues = Map[Column, Literal]( + new Column("geom") -> Literal.ofNull(GeometryType.ofDefault()), + new Column("geog") -> Literal.ofGeospatialWKT( + "POINT (1 2)", + GeographyType.ofDefault())).asJava + + val maxValues = Map[Column, Literal]( + new Column("geom") -> Literal.ofGeospatialWKT( + "POINT (5 6)", + GeometryType.ofDefault()), + new Column("geog") -> Literal.ofNull(GeographyType.ofDefault())).asJava + + val nullCount = Map( + new Column("geom") -> java.lang.Long.valueOf(10L), + new Column("geog") -> java.lang.Long.valueOf(10L)).asJava + + val stats = new DataFileStatistics( + 10, + minValues, + maxValues, + nullCount, + Optional.empty()) + + val json = stats.serializeAsJson(schema) + val expectedJson = + """{ + | "numRecords": 10, + | "minValues": { + | "geom": null, + | "geog": "POINT (1 2)" + | }, + | "maxValues": { + | "geom": "POINT (5 6)", + | "geog": null + | }, + | "nullCount": { + | "geom": 10, + | "geog": 10 + | } + |}""".stripMargin + assert(areJsonNodesEqual(json, expectedJson)) + } + + test("round-trip with POINT Z and POINT ZM variants") { + val schema = new StructType() + .add("geom", GeometryType.ofDefault()) + .add("geog", GeographyType.ofDefault()) + + val minValues = Map( + new Column("geom") -> Literal.ofGeospatialWKT( + "POINT Z (1.0 2.0 3.0)", + GeometryType.ofDefault()), + new Column("geog") -> Literal.ofGeospatialWKT( + "POINT ZM (10.0 20.0 30.0 40.0)", + GeographyType.ofDefault())).asJava + + val maxValues = Map( + new Column("geom") -> Literal.ofGeospatialWKT( + "POINT Z (4.0 5.0 6.0)", + GeometryType.ofDefault()), + new Column("geog") -> Literal.ofGeospatialWKT( + "POINT ZM (40.0 50.0 60.0 70.0)", + GeographyType.ofDefault())).asJava + + val nullCount = Map( + new Column("geom") -> java.lang.Long.valueOf(0L), + new Column("geog") -> java.lang.Long.valueOf(0L)).asJava + + val original = new DataFileStatistics( + 50, + minValues, + maxValues, + nullCount, + Optional.of(true)) + + val json = original.serializeAsJson(schema) + val deserialized = + DataFileStatistics.deserializeFromJson(json, schema) + assert(deserialized.isPresent) + + val stats = deserialized.get() + assert(stats.getMinValues.get(new Column("geom")).getValue == + "POINT Z (1.0 2.0 3.0)") + assert(stats.getMinValues.get(new Column("geog")).getValue == + "POINT ZM (10.0 20.0 30.0 40.0)") + assert(stats.getMaxValues.get(new Column("geom")).getValue == + "POINT Z (4.0 5.0 6.0)") + assert(stats.getMaxValues.get(new Column("geog")).getValue == + "POINT ZM (40.0 50.0 60.0 70.0)") + } + + test("round-trip preserves full double precision (17 digits) in POINT WKT") { + // 17 significant digits is the max needed to distinguish IEEE 754 doubles + val minWkt = "POINT (-122.41941550123456 37.774929501234567)" + val maxWkt = "POINT Z (179.99999999999997 89.999999999999986 12345.678901234567)" + + val schema = new StructType() + .add("geom", GeometryType.ofDefault()) + .add("geom3d", GeometryType.ofDefault()) + + val minValues = Map( + new Column("geom") -> Literal.ofGeospatialWKT( + minWkt, + GeometryType.ofDefault()), + new Column("geom3d") -> Literal.ofGeospatialWKT( + maxWkt, + GeometryType.ofDefault())).asJava + + val maxValues = Map( + new Column("geom") -> Literal.ofGeospatialWKT( + minWkt, + GeometryType.ofDefault()), + new Column("geom3d") -> Literal.ofGeospatialWKT( + maxWkt, + GeometryType.ofDefault())).asJava + + val nullCount = Map( + new Column("geom") -> java.lang.Long.valueOf(0L), + new Column("geom3d") -> java.lang.Long.valueOf(0L)).asJava + + val original = new DataFileStatistics( + 1, + minValues, + maxValues, + nullCount, + Optional.of(true)) + + val json = original.serializeAsJson(schema) + val deserialized = + DataFileStatistics.deserializeFromJson(json, schema) + assert(deserialized.isPresent) + + val stats = deserialized.get() + assert(stats.getMinValues.get(new Column("geom")).getValue === minWkt) + assert(stats.getMaxValues.get(new Column("geom")).getValue === minWkt) + assert(stats.getMinValues.get(new Column("geom3d")).getValue === + maxWkt) + assert(stats.getMaxValues.get(new Column("geom3d")).getValue === + maxWkt) + } + + test("deserializing invalid geospatial stats throws KernelException") { + val schema = new StructType() + .add("geom", GeometryType.ofDefault()) + + val invalidJson = + """|{ + | "numRecords": 10, + | "minValues": { + | "geom": 1234 + | } + |}""".stripMargin + + val exception = intercept[KernelException] { + DataFileStatistics.deserializeFromJson(invalidJson, schema) + } + assert(exception.getMessage.contains("Expected geometry as WKT string but got")) + } + + test("serialize throws when geospatial literal type mismatches schema") { + val schema = new StructType() + .add("geom", GeometryType.ofDefault()) + + val minValues = Map[Column, Literal]( + new Column("geom") -> Literal.ofString("POINT (1 2)")).asJava + + val stats = new DataFileStatistics( + 10, + minValues, + Collections.emptyMap[Column, Literal](), + Collections.emptyMap[Column, java.lang.Long](), + Optional.empty()) + + val exception = intercept[KernelException] { + stats.serializeAsJson(schema) + } + assert(exception.getMessage.contains("Type mismatch")) + assert(exception.getMessage.contains("geom")) + } + + test("deserializing non-POINT geospatial stats throws KernelException") { + val schema = new StructType() + .add("geom", GeometryType.ofDefault()) + + val lineStringJson = + """|{ + | "numRecords": 10, + | "minValues": { + | "geom": "LINESTRING (0 0, 1 1)" + | } + |}""".stripMargin + + val ex1 = intercept[KernelException] { + DataFileStatistics.deserializeFromJson(lineStringJson, schema) + } + assert(ex1.getMessage.contains("Geospatial stats must be a valid POINT WKT")) + + val polygonJson = + """|{ + | "numRecords": 10, + | "minValues": { + | "geom": "POLYGON ((0 0, 1 0, 1 1, 0 0))" + | } + |}""".stripMargin + + val ex2 = intercept[KernelException] { + DataFileStatistics.deserializeFromJson(polygonJson, schema) + } + assert(ex2.getMessage.contains("Geospatial stats must be a valid POINT WKT")) + } + + test("serializing non-POINT geospatial literal throws KernelException") { + val schema = new StructType() + .add("geog", GeographyType.ofDefault()) + + val minValues = Map[Column, Literal]( + new Column("geog") -> Literal.ofGeospatialWKT( + "LINESTRING (0 0, 1 1)", + GeographyType.ofDefault())).asJava + + val stats = new DataFileStatistics( + 10, + minValues, + Collections.emptyMap[Column, Literal](), + Collections.emptyMap[Column, java.lang.Long](), + Optional.empty()) + + val exception = intercept[KernelException] { + stats.serializeAsJson(schema) + } + assert(exception.getMessage.contains( + "Geospatial stats must be a valid POINT WKT")) + } + + test("constructing geospatial literal with null WKT throws") { + intercept[NullPointerException] { + Literal.ofGeospatialWKT(null, GeometryType.ofDefault()) + } + } + + test("deserializing malformed POINT WKT throws KernelException") { + val schema = new StructType() + .add("geom", GeometryType.ofDefault()) + + // POINT with wrong coordinate count + val wrongCoordCount = + """|{ + | "numRecords": 10, + | "minValues": { "geom": "POINT (1 2 3)" } + |}""".stripMargin + val ex1 = intercept[KernelException] { + DataFileStatistics.deserializeFromJson(wrongCoordCount, schema) + } + assert(ex1.getMessage.contains("Geospatial stats must be a valid POINT WKT")) + + // POINT with non-numeric coordinates + val nonNumeric = + """|{ + | "numRecords": 10, + | "minValues": { "geom": "POINT (abc def)" } + |}""".stripMargin + val ex2 = intercept[KernelException] { + DataFileStatistics.deserializeFromJson(nonNumeric, schema) + } + assert(ex2.getMessage.contains("Geospatial stats must be a valid POINT WKT")) + + // POINT with missing parens + val missingParens = + """|{ + | "numRecords": 10, + | "minValues": { "geom": "POINT 1 2" } + |}""".stripMargin + val ex3 = intercept[KernelException] { + DataFileStatistics.deserializeFromJson(missingParens, schema) + } + assert(ex3.getMessage.contains("Geospatial stats must be a valid POINT WKT")) + } + + test("deserialization with missing geospatial stats in JSON") { + val schema = new StructType() + .add("geom", GeometryType.ofDefault()) + .add("geog", GeographyType.ofDefault()) + .add("id", IntegerType.INTEGER) + + // Stats JSON has no min/max for geospatial columns + val json = + """{ + | "numRecords": 100, + | "minValues": { "id": 1 }, + | "maxValues": { "id": 99 }, + | "nullCount": { "id": 0 } + |}""".stripMargin + + val result = DataFileStatistics.deserializeFromJson(json, schema) + assert(result.isPresent) + val stats = result.get() + assert(stats.getNumRecords == 100) + assert(!stats.getMinValues.containsKey(new Column("geom"))) + assert(!stats.getMinValues.containsKey(new Column("geog"))) + assert(stats.getMinValues.get(new Column("id")).getValue == 1) + } + + test("geography coordinate range validation on deserialization") { + val schema = new StructType() + .add("geog", GeographyType.ofDefault()) + + val outOfRange = + """|{ + | "numRecords": 10, + | "minValues": { "geog": "POINT (200 0)" } + |}""".stripMargin + val ex = intercept[KernelException] { + DataFileStatistics.deserializeFromJson(outOfRange, schema) + } + assert(ex.getMessage.contains("out of range")) + } + + test("geography coordinate range validation on serialization") { + val schema = new StructType() + .add("geog", GeographyType.ofDefault()) + + val minValues = Map[Column, Literal]( + new Column("geog") -> Literal.ofGeospatialWKT( + "POINT (200 0)", + GeographyType.ofDefault())).asJava + + val stats = new DataFileStatistics( + 10, + minValues, + Collections.emptyMap[Column, Literal](), + Collections.emptyMap[Column, java.lang.Long](), + Optional.empty()) + + val ex = intercept[KernelException] { + stats.serializeAsJson(schema) + } + assert(ex.getMessage.contains("out of range")) + } + } diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/util/GeometryUtilsSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/util/GeometryUtilsSuite.scala new file mode 100644 index 00000000000..9e19b604e9f --- /dev/null +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/internal/util/GeometryUtilsSuite.scala @@ -0,0 +1,297 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.kernel.internal.util + +import java.util.OptionalDouble + +import io.delta.kernel.internal.util.GeometryUtils.{formatPointWKT, parsePointXY} + +import org.scalatest.funsuite.AnyFunSuite + +class GeometryUtilsSuite extends AnyFunSuite { + + private def checkXY(wkt: String, x: Double, y: Double): Unit = { + val xy = parsePointXY(wkt) + assert(xy(0) === x) + assert(xy(1) === y) + } + + private def checkInvalid(wkt: String, msgFragment: String): Unit = { + withClue(s"Expected exception for input: $wkt") { + val e = intercept[IllegalArgumentException](parsePointXY(wkt)) + assert(e.getMessage.contains(msgFragment)) + } + } + + test("POINT (x y) - basic 2D") { + checkXY("POINT (1.0 2.0)", 1.0, 2.0) + checkXY("POINT (0.0 0.0)", 0.0, 0.0) + checkXY("POINT (-1.5 3.7)", -1.5, 3.7) + } + + test("POINT Z(x y z) - 3D") { + checkXY("POINT Z(1.0 2.0 3.0)", 1.0, 2.0) + checkXY("POINT Z(-1.0 -2.0 -3.0)", -1.0, -2.0) + } + + test("POINT M(x y m) - measured") { + checkXY("POINT M(1.0 2.0 4.0)", 1.0, 2.0) + } + + test("POINT ZM(x y z m) - 4D") { + checkXY("POINT ZM(1.0 2.0 3.0 4.0)", 1.0, 2.0) + checkXY("POINT ZM(0.0 0.0 0.0 0.0)", 0.0, 0.0) + } + + test("POINT EMPTY variants - returns NaN") { + for ( + wkt <- Seq( + "POINT EMPTY", + "POINT Z EMPTY", + "POINT M EMPTY", + "POINT ZM EMPTY", + "point empty", + "point z empty", + "point m empty", + "point zm empty", + "POINT EMPTY", + "POINT Z EMPTY", + "POINT ZM EMPTY") + ) { + withClue(s"$wkt should parse as NaN: ") { + val xy = parsePointXY(wkt) + assert(xy(0).isNaN) + assert(xy(1).isNaN) + } + } + } + + test("whitespace variations") { + // no space before paren + checkXY("POINT(1.0 2.0)", 1.0, 2.0) + checkXY("POINT Z(1.0 2.0 3.0)", 1.0, 2.0) + checkXY("POINT M(1.0 2.0 4.0)", 1.0, 2.0) + checkXY("POINT ZM(1.0 2.0 3.0 4.0)", 1.0, 2.0) + // extra spaces before paren + checkXY("POINT (1.0 2.0)", 1.0, 2.0) + checkXY("POINT Z (1.0 2.0 3.0)", 1.0, 2.0) + checkXY("POINT M (1.0 2.0 4.0)", 1.0, 2.0) + checkXY("POINT ZM (1.0 2.0 3.0 4.0)", 1.0, 2.0) + // extra spaces between modifier and paren + checkXY("POINT Z (1.0 2.0 3.0)", 1.0, 2.0) + checkXY("POINT M (1.0 2.0 4.0)", 1.0, 2.0) + checkXY("POINT ZM (1.0 2.0 3.0 4.0)", 1.0, 2.0) + // extra spaces around/between coordinates + checkXY("POINT ( 1.0 2.0 )", 1.0, 2.0) + checkXY("POINT Z ( 1.0 2.0 3.0 )", 1.0, 2.0) + checkXY("POINT ZM ( 1.0 2.0 3.0 4.0 )", 1.0, 2.0) + } + + test("case insensitivity") { + checkXY("point (1.0 2.0)", 1.0, 2.0) + checkXY("POINT zm(1.0 2.0 3.0 4.0)", 1.0, 2.0) + checkXY("Point Z(1.0 2.0 3.0)", 1.0, 2.0) + } + + test("scientific notation") { + checkXY("POINT (1.5e2 2.5E2)", 150.0, 250.0) + checkXY("POINT (-1.5e2 2.5e-1)", -150.0, 0.25) + checkXY("POINT (1e3 2e3)", 1000.0, 2000.0) + checkXY("POINT (-1E3 -2E3)", -1000.0, -2000.0) + checkXY("POINT (1.23456789e10 -9.87654321e-5)", 1.23456789e10, -9.87654321e-5) + checkXY("POINT Z (1e2 2e2 3e2)", 100.0, 200.0) + } + + test("integer coordinates (no decimal point)") { + checkXY("POINT (1 2)", 1.0, 2.0) + checkXY("POINT (-100 200)", -100.0, 200.0) + checkXY("POINT (0 0)", 0.0, 0.0) + checkXY("POINT Z (1 2 3)", 1.0, 2.0) + checkXY("POINT ZM (1 2 3 4)", 1.0, 2.0) + } + + test("null input throws") { + val e = intercept[NullPointerException](parsePointXY(null)) + assert(e.getMessage.contains("cannot be null")) + } + + test("missing parens throws") { + checkInvalid("POINT 1.0 2.0", "Invalid WKT POINT") + checkInvalid("POINT", "Invalid WKT POINT") + } + + test("wrong coordinate count throws") { + checkInvalid("POINT (1.0 2.0 3.0)", "expects 2 coordinates but got 3") + checkInvalid("POINT Z(1.0 2.0)", "expects 3 coordinates but got 2") + checkInvalid("POINT M(1.0 2.0)", "expects 3 coordinates but got 2") + checkInvalid("POINT ZM(1.0 2.0 3.0)", "expects 4 coordinates but got 3") + } + + test("excess coordinates throws") { + checkInvalid("POINT Z(1.0 2.0 3.0 4.0)", "expects 3 coordinates but got 4") + checkInvalid("POINT M(1.0 2.0 3.0 4.0)", "expects 3 coordinates but got 4") + checkInvalid("POINT ZM(1.0 2.0 3.0 4.0 5.0)", "expects 4 coordinates but got 5") + } + + test("non-space separators are rejected") { + // comma/semicolon create 1 token instead of 2 + checkInvalid("POINT (1.0,2.0)", "expects 2 coordinates but got 1") + checkInvalid("POINT (1.0;2.0)", "expects 2 coordinates but got 1") + } + + test("non-numeric coordinate throws") { + checkInvalid("POINT (1.0 foo)", "Invalid coordinate") + checkInvalid("POINT ZM(1.0 2.0 3.0 bar)", "Invalid coordinate") + } + + test("NaN and Infinity coordinates throw") { + checkInvalid("POINT (NaN 1.0)", "finite numbers") + checkInvalid("POINT (1.0 NaN)", "finite numbers") + checkInvalid("POINT (Infinity 1.0)", "finite numbers") + checkInvalid("POINT (1.0 -Infinity)", "finite numbers") + checkInvalid("POINT Z(1.0 2.0 NaN)", "finite numbers") + } + + test("formatPointWKT - 2D") { + val wkt = formatPointWKT( + 1.0, + 2.0, + OptionalDouble.empty(), + OptionalDouble.empty()) + assert(wkt === "POINT(1.0 2.0)") + val xy = parsePointXY(wkt) + assert(xy(0) === 1.0) + assert(xy(1) === 2.0) + } + + test("formatPointWKT - Z round-trip") { + val wkt = formatPointWKT( + 1.0, + 2.0, + OptionalDouble.of(3.0), + OptionalDouble.empty()) + assert(wkt === "POINT Z (1.0 2.0 3.0)") + val xy = parsePointXY(wkt) + assert(xy(0) === 1.0) + assert(xy(1) === 2.0) + } + + test("formatPointWKT - M round-trip") { + val wkt = formatPointWKT( + 1.0, + 2.0, + OptionalDouble.empty(), + OptionalDouble.of(4.0)) + assert(wkt === "POINT M (1.0 2.0 4.0)") + val xy = parsePointXY(wkt) + assert(xy(0) === 1.0) + assert(xy(1) === 2.0) + } + + test("formatPointWKT - ZM round-trip") { + val wkt = formatPointWKT( + 1.0, + 2.0, + OptionalDouble.of(3.0), + OptionalDouble.of(4.0)) + assert(wkt === "POINT ZM (1.0 2.0 3.0 4.0)") + val xy = parsePointXY(wkt) + assert(xy(0) === 1.0) + assert(xy(1) === 2.0) + } + + test("formatPointWKT - EMPTY round-trips") { + val empty2d = formatPointWKT( + Double.NaN, + Double.NaN, + OptionalDouble.empty(), + OptionalDouble.empty()) + assert(empty2d === "POINT EMPTY") + val xy = parsePointXY(empty2d) + assert(xy(0).isNaN) + assert(xy(1).isNaN) + + val emptyZ = formatPointWKT( + Double.NaN, + Double.NaN, + OptionalDouble.of(Double.NaN), + OptionalDouble.empty()) + assert(emptyZ === "POINT Z EMPTY") + assert(parsePointXY(emptyZ)(0).isNaN) + + val emptyM = formatPointWKT( + Double.NaN, + Double.NaN, + OptionalDouble.empty(), + OptionalDouble.of(Double.NaN)) + assert(emptyM === "POINT M EMPTY") + assert(parsePointXY(emptyM)(0).isNaN) + + val emptyZM = formatPointWKT( + Double.NaN, + Double.NaN, + OptionalDouble.of(Double.NaN), + OptionalDouble.of(Double.NaN)) + assert(emptyZM === "POINT ZM EMPTY") + assert(parsePointXY(emptyZM)(0).isNaN) + } + + test("formatPointWKT - round-trip negative coords") { + val wkt = formatPointWKT( + -180.0, + -90.0, + OptionalDouble.empty(), + OptionalDouble.empty()) + val xy = parsePointXY(wkt) + assert(xy(0) === -180.0) + assert(xy(1) === -90.0) + } + + // Geography coordinate range validation tests + + test("validateGeographyPointWKT - valid coordinates") { + GeometryUtils.validateGeographyPointWKT("POINT (0 0)") + GeometryUtils.validateGeographyPointWKT("POINT (-180 -90)") + GeometryUtils.validateGeographyPointWKT("POINT (180 90)") + GeometryUtils.validateGeographyPointWKT("POINT Z (10 20 100)") + GeometryUtils.validateGeographyPointWKT("POINT EMPTY") + } + + test("validateGeographyPointWKT - longitude out of range") { + val e = intercept[io.delta.kernel.exceptions.KernelException] { + GeometryUtils.validateGeographyPointWKT("POINT (181 0)") + } + assert(e.getMessage.contains("out of range")) + + val e2 = intercept[io.delta.kernel.exceptions.KernelException] { + GeometryUtils.validateGeographyPointWKT("POINT (-181 0)") + } + assert(e2.getMessage.contains("out of range")) + } + + test("validateGeographyPointWKT - latitude out of range") { + val e = intercept[io.delta.kernel.exceptions.KernelException] { + GeometryUtils.validateGeographyPointWKT("POINT (0 91)") + } + assert(e.getMessage.contains("out of range")) + + val e2 = intercept[io.delta.kernel.exceptions.KernelException] { + GeometryUtils.validateGeographyPointWKT("POINT (0 -91)") + } + assert(e2.getMessage.contains("out of range")) + } + +} diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/types/FieldMetadataSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/types/FieldMetadataSuite.scala index 81a3dc20ff2..35e4dc2b77d 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/types/FieldMetadataSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/types/FieldMetadataSuite.scala @@ -326,4 +326,144 @@ class FieldMetadataSuite extends AnyFunSuite { } assert(e.getMessage == "keys must not be null") } + + test("equals validates all keys and values") { + val meta1 = FieldMetadata.builder() + .putString("collation", "UTF8_BINARY") + .putString("otherKey", "same") + .putLongArray("arr", Seq[java.lang.Long](1L, 2L).toArray) + .build() + + val meta2 = FieldMetadata.builder() + .putString("collation", "UTF8_BINARY") + .putString("otherKey", "same") + .putLongArray("arr", Seq[java.lang.Long](1L, 2L).toArray) + .build() + + val meta3 = FieldMetadata.builder() + .putString("collation", "EN_CI") // different -> should not be equal + .putString("otherKey", "same") + .putLongArray("arr", Seq[java.lang.Long](1L, 2L).toArray) + .build() + + assertThat(meta1.equals(meta2)).isTrue + assertThat(meta1.equals(meta3)).isFalse + } + + test("equals handles arrays properly") { + val longs: Seq[java.lang.Long] = Seq(1L, 2L, 3L) + val doubles: Seq[java.lang.Double] = Seq(1.0, 2.0, 3.0) + val booleans: Seq[java.lang.Boolean] = Seq(true, false, true) + val strings: Seq[java.lang.String] = Seq("x", "y", "z") + val stringsWithNulls: Seq[java.lang.String] = Seq("a", null, "b") + val inner1 = FieldMetadata.builder().putBoolean("k", true).build() + val inner2 = FieldMetadata.builder().putBoolean("k", true).build() + + val meta1 = FieldMetadata.builder() + .putLongArray("longArrayKey", longs.toArray) + .putDoubleArray("doubleArrayKey", doubles.toArray) + .putBooleanArray("booleanArrayKey", booleans.toArray) + .putStringArray("stringArrayKey", strings.toArray) + .putStringArray("stringArrayWithNulls", stringsWithNulls.toArray) + .putFieldMetadataArray("fieldMetadataArrayKey", Array(inner1)) + .build() + + val meta2 = FieldMetadata.builder() + .putLongArray("longArrayKey", longs.toArray) + .putDoubleArray("doubleArrayKey", doubles.toArray) + .putBooleanArray("booleanArrayKey", booleans.toArray) + .putStringArray("stringArrayKey", strings.toArray) + .putStringArray("stringArrayWithNulls", stringsWithNulls.toArray) + .putFieldMetadataArray("fieldMetadataArrayKey", Array(inner2)) + .build() + + assertThat(meta1.equals(meta2)).isTrue + + // Change one array element to ensure inequality is detected + val meta3 = FieldMetadata.builder() + .putLongArray("longArrayKey", Seq[java.lang.Long](1L, 2L, 99L).toArray) + .putDoubleArray("doubleArrayKey", doubles.toArray) + .putBooleanArray("booleanArrayKey", booleans.toArray) + .putStringArray("stringArrayKey", strings.toArray) + .putStringArray("stringArrayWithNulls", stringsWithNulls.toArray) + .putFieldMetadataArray("fieldMetadataArrayKey", Array(inner2)) + .build() + + assertThat(meta1.equals(meta3)).isFalse + } + + test("equals handles case where only one side has the key") { + val meta1 = FieldMetadata.builder() + .putString("collation", "EN_CI") + .putString("common", "v") + .build() + val meta2 = FieldMetadata.builder() + .putString("common", "v") + .build() + + assertThat(meta1.equals(meta2)).isFalse + assertThat(meta2.equals(meta1)).isFalse + } + + test("equals handles entries with null value") { + val meta1 = FieldMetadata.builder() + .putString("nullableKey", null) + .putString("same", "x") + .build() + val meta2 = FieldMetadata.builder() + .putString("nullableKey", null) + .putString("same", "x") + .build() + val meta3 = FieldMetadata.builder() + .putString("nullableKey", "value") + .putString("same", "x") + .build() + + assertThat(meta1.equals(meta2)).isTrue + assertThat(meta1.equals(meta3)).isFalse + } + + test("equals handles entries with null key") { + val meta1 = FieldMetadata.builder() + .putString(null, "A") + .putString("same", "x") + .build() + val meta2 = FieldMetadata.builder() + .putString(null, "A") + .putString("same", "x") + .build() + val meta3 = FieldMetadata.builder() + .putString(null, "B") + .putString("same", "x") + .build() + val meta4 = FieldMetadata.builder() + .putString("same", "x") + .build() + + // same key/value pairs -> equal + assertThat(meta1.equals(meta2)).isTrue + // different values under null key -> unequal + assertThat(meta1.equals(meta3)).isFalse + // one side missing the null key -> unequal + assertThat(meta1.equals(meta4)).isFalse + } + + test("equals handles entry with null key and null value") { + val meta1 = FieldMetadata.builder() + .putString(null, null) + .putString("same", "x") + .build() + val meta2 = FieldMetadata.builder() + .putString(null, null) + .putString("same", "x") + .build() + + assertThat(meta1.equals(meta2)).isTrue + } + + test("equals returns false for null or different class") { + val meta = FieldMetadata.builder().putString("k", "v").build() + assertThat(meta.equals(null)).isFalse + assertThat(meta.equals("not-metadata")).isFalse + } } diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/types/StructFieldSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/types/StructFieldSuite.scala index 4385175cb66..832e8c3a2e8 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/types/StructFieldSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/types/StructFieldSuite.scala @@ -266,18 +266,13 @@ class StructFieldSuite extends AnyFunSuite { new StructField( "mapField", new MapType( - new StructField("key", new StringType("ICU.DE_DE"), false), + new StructField("key", StringType.STRING, false), new StructField("value", LongType.LONG, true).withTypeChanges( Seq( new TypeChange(ShortType.SHORT, IntegerType.INTEGER), new TypeChange(IntegerType.INTEGER, LongType.LONG)).asJava)), false), FieldMetadata.builder() - .putFieldMetadata( - COLLATIONS_METADATA_KEY, - FieldMetadata.builder() - .putString("mapField.key", "ICU.DE_DE") - .build()) .putFieldMetadataArray( DELTA_TYPE_CHANGES_KEY, Array( diff --git a/kernel/kernel-api/src/test/scala/io/delta/kernel/types/TypesSuite.scala b/kernel/kernel-api/src/test/scala/io/delta/kernel/types/TypesSuite.scala index 3989c46e62f..94372e058e4 100644 --- a/kernel/kernel-api/src/test/scala/io/delta/kernel/types/TypesSuite.scala +++ b/kernel/kernel-api/src/test/scala/io/delta/kernel/types/TypesSuite.scala @@ -64,4 +64,50 @@ class TypesSuite extends AnyFunSuite { assert(dataType.isNested(), s"Expected $dataType to be nested") } } + + test("MapType constructor throws IllegalArgumentException for collated StringType keys") { + // Test multiple collation providers to ensure all non-default collations are rejected + Seq("SPARK.UTF8_LCASE", "ICU.UNICODE_CI").foreach { collation => + val collatedString = new StringType(collation) + + // 3-arg constructor + val ex1 = intercept[IllegalArgumentException] { + new MapType(collatedString, IntegerType.INTEGER, false) + } + assert(ex1.getMessage.contains("does not support collated string types as keys")) + assert(ex1.getMessage.contains("UTF8_BINARY")) + assert( + ex1.getMessage.contains(collatedString.toString), + s"Error message should include the found type but was: ${ex1.getMessage}") + + // 2-arg StructField constructor + val ex2 = intercept[IllegalArgumentException] { + new MapType( + new StructField("key", collatedString, false), + new StructField("value", IntegerType.INTEGER, true)) + } + assert(ex2.getMessage.contains("does not support collated string types as keys")) + assert( + ex2.getMessage.contains(collatedString.toString), + s"Error message should include the found type but was: ${ex2.getMessage}") + } + } + + test("MapType allows default UTF8_BINARY StringType keys and non-string keys") { + val map1 = new MapType(StringType.STRING, IntegerType.INTEGER, false) + assert(map1.getKeyType === StringType.STRING) + + val utf8BinaryString = new StringType("SPARK.UTF8_BINARY") + val map2 = new MapType(utf8BinaryString, IntegerType.INTEGER, false) + assert(map2.getKeyType === utf8BinaryString) + + val map3 = new MapType(IntegerType.INTEGER, StringType.STRING, true) + assert(map3.getKeyType === IntegerType.INTEGER) + } + + test("MapType allows collated StringType as values") { + val collatedString = new StringType("SPARK.UTF8_LCASE") + val map = new MapType(StringType.STRING, collatedString, true) + assert(map.getValueType === collatedString) + } } diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/DefaultJsonRow.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/DefaultJsonRow.java index fd807992fa9..fd9324da195 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/DefaultJsonRow.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/DefaultJsonRow.java @@ -350,6 +350,11 @@ public ColumnVector getValues() { }; } + if (dataType instanceof GeometryType || dataType instanceof GeographyType) { + throwIfTypeMismatch("string", jsonValue.isTextual(), jsonValue); + return jsonValue.asText(); + } + throw new UnsupportedOperationException( String.format("Unsupported DataType %s for RootNode %s", dataType, jsonValue)); } diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DefaultGenericVector.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DefaultGenericVector.java index 132e6cd66d6..fcb3ae4889d 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DefaultGenericVector.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DefaultGenericVector.java @@ -117,7 +117,12 @@ public double getDouble(int rowId) { @Override public String getString(int rowId) { assertValidRowId(rowId); - throwIfUnsafeAccess(StringType.class, "string"); + if (!(dataType instanceof StringType + || dataType instanceof GeometryType + || dataType instanceof GeographyType)) { + throw new UnsupportedOperationException( + String.format("Trying to access a `string` value from vector of type `%s`", dataType)); + } return (String) rowIdToValueAccessor.apply(rowId); } diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java index ab5937afbb7..f886bddb05f 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java @@ -30,6 +30,7 @@ import io.delta.kernel.defaults.internal.data.vector.DefaultConstantVector; import io.delta.kernel.engine.ExpressionHandler; import io.delta.kernel.expressions.*; +import io.delta.kernel.internal.util.GeometryUtils; import io.delta.kernel.types.*; import java.util.*; import java.util.stream.Collectors; @@ -361,6 +362,37 @@ ExpressionTransformResult visitIn(In in) { return new ExpressionTransformResult(transformedExpression, BooleanType.BOOLEAN); } + @Override + ExpressionTransformResult visitStGeometryBoxesIntersectOnStats(Predicate predicate) { + List children = + predicate.getChildren().stream().map(this::visit).collect(Collectors.toList()); + checkArgument( + children.size() == 4, + "ST_GEOMETRY_BOXES_INTERSECT_ON_STATS expects 4 children but got %d", + children.size()); + // All 4 children must be GeometryType. + // Children 0,1 are stats columns, children 2,3 are query literals. + DataType child0Type = children.get(0).outputType; + checkArgument( + child0Type instanceof GeometryType, + "ST_GEOMETRY_BOXES_INTERSECT_ON_STATS child 0 must be " + "geometry type, got %s", + child0Type); + for (int i = 1; i < 4; i++) { + checkArgument( + child0Type.equals(children.get(i).outputType), + "ST_GEOMETRY_BOXES_INTERSECT_ON_STATS child %d type %s " + + "doesn't match child 0 type %s", + i, + children.get(i).outputType, + child0Type); + } + return new ExpressionTransformResult( + new Predicate( + "ST_GEOMETRY_BOXES_INTERSECT_ON_STATS", + children.stream().map(c -> c.expression).collect(Collectors.toList())), + BooleanType.BOOLEAN); + } + private Predicate validateIsPredicate( Expression baseExpression, ExpressionTransformResult result) { checkArgument( @@ -554,7 +586,9 @@ ColumnVector visitLiteral(Literal literal) { || dataType instanceof DecimalType || dataType instanceof DateType || dataType instanceof TimestampType - || dataType instanceof TimestampNTZType) { + || dataType instanceof TimestampNTZType + || dataType instanceof GeometryType + || dataType instanceof GeographyType) { return new DefaultConstantVector(dataType, input.getSize(), literal.getValue()); } @@ -747,6 +781,38 @@ ColumnVector visitIn(In in) { in.getChildren().stream().map(this::visit).collect(toList())); } + @Override + ColumnVector visitStGeometryBoxesIntersectOnStats(Predicate predicate) { + List children = predicate.getChildren(); + ColumnVector leftMin = visit(children.get(0)); + ColumnVector leftMax = visit(children.get(1)); + ColumnVector rightMin = visit(children.get(2)); + ColumnVector rightMax = visit(children.get(3)); + int numRows = input.getSize(); + boolean[] result = new boolean[numRows]; + boolean[] nullability = new boolean[numRows]; + for (int rowId = 0; rowId < numRows; rowId++) { + if (leftMin.isNullAt(rowId) + || leftMax.isNullAt(rowId) + || rightMin.isNullAt(rowId) + || rightMax.isNullAt(rowId)) { + nullability[rowId] = true; + continue; + } + double[] lMin = GeometryUtils.parsePointXY(leftMin.getString(rowId)); + double[] lMax = GeometryUtils.parsePointXY(leftMax.getString(rowId)); + double[] rMin = GeometryUtils.parsePointXY(rightMin.getString(rowId)); + double[] rMax = GeometryUtils.parsePointXY(rightMax.getString(rowId)); + result[rowId] = boxesIntersect(lMin, lMax, rMin, rMax); + } + return new DefaultBooleanVector(numRows, Optional.of(nullability), result); + } + + private static boolean boxesIntersect( + double[] lMin, double[] lMax, double[] rMin, double[] rMax) { + return lMax[0] >= rMin[0] && rMax[0] >= lMin[0] && lMax[1] >= rMin[1] && rMax[1] >= lMin[1]; + } + /** * Utility method to evaluate inputs to the binary input expression. Also validates the * evaluated expression result {@link ColumnVector}s are of the same size. diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/ExpressionVisitor.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/ExpressionVisitor.java index 468dd70225d..3daac18409f 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/ExpressionVisitor.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/ExpressionVisitor.java @@ -74,6 +74,8 @@ abstract class ExpressionVisitor { abstract R visitIn(In in); + abstract R visitStGeometryBoxesIntersectOnStats(Predicate predicate); + final R visit(Expression expression) { if (expression instanceof PartitionValueExpression) { return visitPartitionValue((PartitionValueExpression) expression); @@ -144,6 +146,9 @@ private R visitScalarExpression(ScalarExpression expression) { } else { return visitIn(new In(children.get(0), children.subList(1, children.size()))); } + case "ST_GEOMETRY_BOXES_INTERSECT_ON_STATS": + return visitStGeometryBoxesIntersectOnStats( + createPredicate(name, children, collationIdentifier)); default: throw new UnsupportedOperationException( String.format("Scalar expression `%s` is not supported.", name)); diff --git a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/CommitIcebergActionSuite.scala b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/CommitIcebergActionSuite.scala index 90d8afc759e..1020a843a73 100644 --- a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/CommitIcebergActionSuite.scala +++ b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/CommitIcebergActionSuite.scala @@ -54,6 +54,19 @@ trait AbstractCommitIcebergActionSuite extends AnyFunSuite { self: AbstractWrite private val tblPropertiesIcebergWriterCompatV3Enabled = Map( TableConfig.ICEBERG_WRITER_COMPAT_V3_ENABLED.getKey -> "true") + /** Helper to create a transaction, generate a single action, and commit it. */ + private def commitSingleAction( + engine: Engine, + tablePath: String, + actionFn: Transaction => Row): Unit = { + val txn = getUpdateTxn(engine, tablePath, maxRetries = 0) + val action = actionFn(txn) + commitTransaction( + txn, + engine, + inMemoryIterable(toCloseableIterator(Seq(action).asJava.iterator()))) + } + private def createIcebergCompatAction( actionType: String, // "ADD" or "REMOVE" version: String, @@ -629,4 +642,62 @@ trait AbstractCommitIcebergActionSuite extends AnyFunSuite { self: AbstractWrite } } } + + test("V3: baseRowId and defaultRowCommitVersion are forwarded in remove actions") { + withTempDirAndEngine { (tablePath, engine) => + createEmptyTable( + engine, + tablePath, + testSchema, + tableProperties = tblPropertiesIcebergWriterCompatV3Enabled) + + // Append 1 add so we have something to remove + commitSingleAction( + engine, + tablePath, + createIcebergCompatAction( + "ADD", + "V3", + _, + engine, + generateDataFileStatus(tablePath, "file1.parquet"), + true)) + + // Remove with explicit baseRowId and defaultRowCommitVersion + val expectedBaseRowId = 42L + val expectedDefaultRowCommitVersion = 7L + commitSingleAction( + engine, + tablePath, + txn => + GenerateIcebergCompatActionUtils.generateIcebergCompatWriterV3RemoveAction( + txn.getTransactionState(engine), + generateDataFileStatus(tablePath, "file1.parquet"), + Collections.emptyMap(), + true, // dataChange + Optional.of[java.lang.Long](expectedBaseRowId), + Optional.of[java.lang.Long](expectedDefaultRowCommitVersion), + Optional.empty(), // deletionVectorDescriptor + Optional.of( + TransactionStateRow.getPhysicalSchema(txn.getTransactionState(engine))))) + + // Read back the remove action and verify the values are preserved + val rows = Table.forPath(engine, tablePath).asInstanceOf[TableImpl] + .getChanges(engine, 2, 2, Set(DeltaAction.REMOVE).asJava) + .toSeq + .flatMap(_.getRows.toSeq) + .filterNot(row => row.isNullAt(row.getSchema.indexOf("remove"))) + + assert(rows.size == 1) + val removeFile = new RemoveFile(rows.head.getStruct(rows.head.getSchema.indexOf("remove"))) + assert( + removeFile.getBaseRowId.isPresent, + "baseRowId should be present when caller provides it") + assert(removeFile.getBaseRowId.get == expectedBaseRowId) + assert( + removeFile.getDefaultRowCommitVersion.isPresent, + "defaultRowCommitVersion should be present when caller provides it") + assert(removeFile.getDefaultRowCommitVersion.get == expectedDefaultRowCommitVersion) + } + } } diff --git a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaReplaceTableSuite.scala b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaReplaceTableSuite.scala index 244e6041903..63f42d4f595 100644 --- a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaReplaceTableSuite.scala +++ b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaReplaceTableSuite.scala @@ -120,6 +120,7 @@ abstract class DeltaReplaceTableSuite extends DeltaReplaceTableSuiteBase { } } + /* TODO: re-enable when other table-features are unsupported test("Cannot enable a feature that Kernel does not support") { withTempDirAndEngine { (tablePath, engine) => createInitialTable(engine, tablePath) @@ -132,7 +133,9 @@ abstract class DeltaReplaceTableSuite extends DeltaReplaceTableSuiteBase { }.getMessage.contains("Unsupported Delta writer feature")) } } + */ + /* TODO: re-enable when other table-features are unsupported test("Cannot replace a table with a protocol Kernel does not support") { withTempDirAndEngine { (tablePath, engine) => spark.sql( @@ -148,6 +151,7 @@ abstract class DeltaReplaceTableSuite extends DeltaReplaceTableSuiteBase { }.getMessage.contains("Unsupported Delta writer feature")) } } + */ test("Must provide a schema for replace table transaction") { withTempDirAndEngine { (tablePath, engine) => diff --git a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaTableFeaturesSuite.scala b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaTableFeaturesSuite.scala index f2c6d6b0e4a..28fe7d300f0 100644 --- a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaTableFeaturesSuite.scala +++ b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaTableFeaturesSuite.scala @@ -408,6 +408,83 @@ trait DeltaTableFeaturesSuiteBase extends AnyFunSuite with AbstractWriteUtils { } /* ---- End: type widening tests ---- */ + /* ---- Start: variant shredding tests ---- */ + test("only variantShredding feature is enabled when metadata supports it: new table") { + withTempDirAndEngine { (tablePath, engine) => + createEmptyTable( + engine, + tablePath = tablePath, + schema = testSchema, + tableProperties = Map("delta.enableVariantShredding" -> "true")) + + val protocolV0 = getProtocol(engine, tablePath) + assert(!protocolV0.supportsFeature(TableFeatures.VARIANT_SHREDDING_PREVIEW_RW_FEATURE)) + assert(protocolV0.supportsFeature(TableFeatures.VARIANT_SHREDDING_RW_FEATURE)) + + updateTableMetadata( + engine = engine, + tablePath = tablePath, + tableProperties = Map("delta.enableVariantShredding" -> "true")) + val protocolV1 = getProtocol(engine, tablePath) + assert(protocolV1 === protocolV0) + } + } + + test("only variantShredding feature is enabled when new metadata supports it: existing table") { + withTempDirAndEngine { (tablePath, engine) => + createEmptyTable(engine, tablePath = tablePath, schema = testSchema) + val protocolV0 = getProtocol(engine, tablePath) + assert(!protocolV0.supportsFeature(TableFeatures.VARIANT_SHREDDING_PREVIEW_RW_FEATURE)) + assert(!protocolV0.supportsFeature(TableFeatures.VARIANT_SHREDDING_RW_FEATURE)) + + updateTableMetadata( + engine = engine, + tablePath = tablePath, + tableProperties = Map("delta.enableVariantShredding" -> "true")) + val protocolV1 = getProtocol(engine, tablePath) + assert(!protocolV1.supportsFeature(TableFeatures.VARIANT_SHREDDING_PREVIEW_RW_FEATURE)) + assert(protocolV1.supportsFeature(TableFeatures.VARIANT_SHREDDING_RW_FEATURE)) + } + } + + test("variantShredding-preview in existing table is respected") { + withTempDirAndEngine { (tablePath, engine) => + createEmptyTable( + engine, + tablePath = tablePath, + schema = testSchema, + tableProperties = Map("delta.feature.variantShredding-preview" -> "supported")) + + val protocolV0 = getProtocol(engine, tablePath) + require(protocolV0.supportsFeature(TableFeatures.VARIANT_SHREDDING_PREVIEW_RW_FEATURE)) + require(!protocolV0.supportsFeature(TableFeatures.VARIANT_SHREDDING_RW_FEATURE)) + + updateTableMetadata( + engine = engine, + tablePath = tablePath, + tableProperties = Map("delta.enableVariantShredding" -> "true")) + val protocolV1 = getProtocol(engine, tablePath) + assert(protocolV1.supportsFeature(TableFeatures.VARIANT_SHREDDING_PREVIEW_RW_FEATURE)) + assert(!protocolV1.supportsFeature(TableFeatures.VARIANT_SHREDDING_RW_FEATURE)) + } + } + test("both variantShredding and variantShredding-preview can coexist") { + withTempDirAndEngine { (tablePath, engine) => + createEmptyTable( + engine, + tablePath = tablePath, + schema = testSchema, + tableProperties = Map( + "delta.enableVariantShredding" -> "true", + "delta.feature.variantShredding-preview" -> "supported")) + + val protocol = getProtocol(engine, tablePath) + assert(protocol.supportsFeature(TableFeatures.VARIANT_SHREDDING_RW_FEATURE)) + assert(protocol.supportsFeature(TableFeatures.VARIANT_SHREDDING_PREVIEW_RW_FEATURE)) + } + } + /* ---- End: variant shredding tests ---- */ + /* ---- Start: materialize partition columns tests ---- */ test("materialize partition columns in parquet schema when feature is enabled") { withTempDirAndEngine { (tablePath, engine) => diff --git a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaTableWritesSuite.scala b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaTableWritesSuite.scala index 92928d8c645..89475a850c5 100644 --- a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaTableWritesSuite.scala +++ b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaTableWritesSuite.scala @@ -17,7 +17,7 @@ package io.delta.kernel.defaults import java.io.File import java.nio.file.Files -import java.util.{Locale, Optional} +import java.util.{Locale, Optional, UUID} import scala.collection.immutable.Seq import scala.jdk.CollectionConverters._ @@ -37,10 +37,12 @@ import io.delta.kernel.expressions.{Column, Literal} import io.delta.kernel.expressions.Literal._ import io.delta.kernel.internal.{ScanImpl, SnapshotImpl, TableConfig} import io.delta.kernel.internal.checkpoints.CheckpointerSuite.selectSingleElement +import io.delta.kernel.internal.data.GenericRow import io.delta.kernel.internal.table.SnapshotBuilderImpl import io.delta.kernel.internal.types.DataTypeJsonSerDe import io.delta.kernel.internal.util.{Clock, JsonUtils} import io.delta.kernel.internal.util.SchemaUtils.casePreservingPartitionColNames +import io.delta.kernel.internal.util.Utils.toCloseableIterator import io.delta.kernel.shaded.com.fasterxml.jackson.databind.node.ObjectNode import io.delta.kernel.transaction.DataLayoutSpec import io.delta.kernel.types._ @@ -1031,6 +1033,132 @@ abstract class AbstractDeltaTableWritesSuite extends AnyFunSuite with AbstractWr } } + test("create table with collated columns - collations writer feature in protocol") { + Seq( + new StructType() + .add("id", IntegerType.INTEGER) + .add("val", new StringType("ICU.UNICODE")), + new StructType() + .add("id", IntegerType.INTEGER) + .add("val", new StringType("ICU.SR_CYRL_SRB")), + new StructType() + .add("id", IntegerType.INTEGER) + .add( + "info", + new StructType() + .add("name", new StringType("ICU.UNICODE.74.1"))), + new StructType() + .add("id", IntegerType.INTEGER) + .add("tags", new ArrayType(new StringType("SPARK.UTF8_LCASE"), true)), + new StructType() + .add("id", IntegerType.INTEGER) + .add( + "props", + new MapType(StringType.STRING, new StringType("ICU.UNICODE"), true))).foreach { schema => + withTempDirAndEngine { (tblPath, engine) => + appendData( + engine, + tblPath, + isNewTable = true, + schema, + data = Seq.empty) + + val protocol = getProtocol(engine, tblPath) + assert(protocol.getWriterFeatures.contains("collations")) + } + } + } + + test("create table without collated columns - collations feature NOT in protocol") { + withTempDirAndEngine { (tblPath, engine) => + val schema = new StructType() + .add("id", IntegerType.INTEGER) + .add("name", StringType.STRING) // default UTF8_BINARY collation + + appendData( + engine, + tblPath, + isNewTable = true, + schema, + data = Seq.empty) + + val protocol = getProtocol(engine, tblPath) + assert( + !protocol.getWriterFeatures.contains("collations"), + "Should NOT have 'collations' writer feature when schema has no custom collations") + } + } + + test("write to golden table with collated columns (utf8_binary, utf8_lcase, unicode)") { + Seq( + ("collations-table", "collations"), + ("collations-preview-table", "collations-preview")).foreach { + case (collationTable, feature) => + withTempDirAndEngine { (tblPath, engine) => + copyTable(collationTable, tblPath) + + val utf8Lcase = new StringType("SPARK.UTF8_LCASE") + val unicode = new StringType("ICU.UNICODE") + + val schema = new StructType() + .add("id", IntegerType.INTEGER) + .add("utf8_binary_col", StringType.STRING) + .add("utf8_lcase_col", utf8Lcase) + .add("unicode_col", unicode) + + // Verify golden table schema has expected collations + val goldenMetadata = getMetadata(engine, tblPath) + val goldenSchema = + DataTypeJsonSerDe.deserializeStructType(goldenMetadata.getSchemaString()) + assert(goldenSchema === schema) + + // Verify protocol has the expected collations writer feature + val protocol = getProtocol(engine, tblPath) + assert(protocol.getWriterFeatures.contains(feature)) + + // First append: data batch uses matching collations + val data1 = generateData(schema, Seq.empty, Map.empty, batchSize = 10, numBatches = 1) + + val commitResult1 = appendData( + engine, + tblPath, + data = Seq(Map.empty[String, Literal] -> data1)) + + verifyCommitResult(commitResult1, expVersion = 1, expIsReadyForCheckpoint = false) + verifyCommitInfo(tblPath, version = 1, partitionCols = null) + verifyWrittenContent(tblPath, schema, data1.flatMap(_.toTestRows)) + + // Second append: data batch uses different collations than the table schema + val diffCollationSchema = new StructType() + .add("id", IntegerType.INTEGER) + .add("utf8_binary_col", new StringType("ICU.UNICODE.75.1")) + .add("utf8_lcase_col", new StringType("SPARK.UTF8_BINARY")) + .add("unicode_col", utf8Lcase) + + val data2 = + generateData(diffCollationSchema, Seq.empty, Map.empty, batchSize = 5, numBatches = 1) + + val commitResult2 = appendData( + engine, + tblPath, + data = Seq(Map.empty[String, Literal] -> data2)) + + verifyCommitResult(commitResult2, expVersion = 2, expIsReadyForCheckpoint = false) + verifyCommitInfo(tblPath, version = 2, partitionCols = null) + verifyWrittenContent(tblPath, schema, (data1 ++ data2).flatMap(_.toTestRows)) + + // Verify the schema is preserved with collations after writes + val metadata = getMetadata(engine, tblPath) + val parsedSchema = DataTypeJsonSerDe.deserializeStructType(metadata.getSchemaString()) + assert(parsedSchema === schema) + + // After writes, the feature flags must not have changed + val protocolAfterWrite = getProtocol(engine, tblPath) + assert(protocolAfterWrite.getWriterFeatures.contains(feature)) + } + } + } + /////////////////////////////////////////////////////////////////////////// // Create table and insert data tests (CTAS & INSERT) /////////////////////////////////////////////////////////////////////////// @@ -1833,6 +1961,108 @@ abstract class AbstractDeltaTableWritesSuite extends AnyFunSuite with AbstractWr } } + /////////////////////////////////////////////////////////////////////////// + // Change Data Feed (CDF) tests + /////////////////////////////////////////////////////////////////////////// + + private def createAddFileRow( + path: String = s"part-${UUID.randomUUID()}.parquet", + dataChange: Boolean = true): Row = { + import io.delta.kernel.internal.actions.{AddFile, SingleAction} + import io.delta.kernel.internal.util.PartitionUtils + + val partitionValues = PartitionUtils.serializePartitionMap( + java.util.Collections.emptyMap[String, Literal]()) + + val addFileRow = AddFile.createAddFileRow( + testSchema, + path, + partitionValues, + 100L, // size + System.currentTimeMillis(), // modificationTime + dataChange, + Optional.empty(), // deletionVector + Optional.empty(), // tags + Optional.empty(), // baseRowId + Optional.empty(), // defaultRowCommitVersion + Optional.empty() // stats + ) + + SingleAction.createAddFileSingleAction(addFileRow) + } + + // Helper to create a mock RemoveFile action row + private def createRemoveFileRow( + path: String, + dataChange: Boolean = true): Row = { + import io.delta.kernel.internal.actions.{RemoveFile, SingleAction} + + val removeFileRow = new GenericRow( + RemoveFile.FULL_SCHEMA, + Map[Integer, Object]( + Integer.valueOf(RemoveFile.FULL_SCHEMA.indexOf("path")) -> path, + Integer.valueOf(RemoveFile.FULL_SCHEMA.indexOf("deletionTimestamp")) -> Long.box( + System.currentTimeMillis()), + Integer.valueOf(RemoveFile.FULL_SCHEMA.indexOf("dataChange")) -> Boolean.box(dataChange), + Integer.valueOf(RemoveFile.FULL_SCHEMA.indexOf("size")) -> Long.box(100L)).asJava) + + SingleAction.createRemoveFileSingleAction(removeFileRow) + } + + // Test cases: (description, actions, shouldSucceed) + val cdfTestCases: Seq[(String, Seq[Row], Boolean)] = Seq( + ("add with dataChange=true", Seq(createAddFileRow(dataChange = true)), true), + ("add with dataChange=false", Seq(createAddFileRow(dataChange = false)), true), + ( + "multiple adds with dataChange=true", + Seq(createAddFileRow(dataChange = true), createAddFileRow(dataChange = true)), + true), + ( + "multiple removes with dataChange=true", { + val path1 = "file1.parquet" + val path2 = "file2.parquet" + Seq( + createAddFileRow(path1, dataChange = false), + createAddFileRow(path2, dataChange = false), + createRemoveFileRow(path1, dataChange = true), + createRemoveFileRow(path2, dataChange = true)) + }, + true), + ( + "add dataChange=true, remove dataChange=false", { + val path = "file1.parquet" + Seq( + createAddFileRow(path, dataChange = true), + createRemoveFileRow(path, dataChange = false)) + }, + true), + ( + "add dataChange=true, remove dataChange=true", { + val path = "file1.parquet" + Seq(createAddFileRow(path, dataChange = true), createRemoveFileRow(path, dataChange = true)) + }, + false)) + + cdfTestCases.foreach { case (desc, actions, shouldSucceed) => + test(s"CDF-enabled table: $desc - ${if (shouldSucceed) "succeeds" else "fails"}") { + withTempDirAndEngine { (tablePath, engine) => + val tableProps = Map(TableConfig.CHANGE_DATA_FEED_ENABLED.getKey -> "true") + val txn = getCreateTxn(engine, tablePath, testSchema, tableProperties = tableProps) + + val actionsIterable = inMemoryIterable(toCloseableIterator(actions.asJava.iterator())) + + if (shouldSucceed) { + val result = commitTransaction(txn, engine, actionsIterable) + assert(result.getVersion === 0) + } else { + intercept[KernelException] { + commitTransaction(txn, engine, actionsIterable) + } + } + } + } + } + def removeTimestampNtzTypeColumns(structType: StructType): StructType = { def process(dataType: DataType): Option[DataType] = dataType match { case a: ArrayType => diff --git a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/GeometryDataSkippingSuite.scala b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/GeometryDataSkippingSuite.scala new file mode 100644 index 00000000000..6fbad7b3bf9 --- /dev/null +++ b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/GeometryDataSkippingSuite.scala @@ -0,0 +1,349 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.kernel.defaults + +import java.util.{Collections, Optional} + +import scala.collection.JavaConverters._ +import scala.collection.immutable.Seq + +import io.delta.kernel._ +import io.delta.kernel.defaults.utils.WriteUtils +import io.delta.kernel.expressions.{Column, Literal, StGeometryBoxesIntersect} +import io.delta.kernel.internal.util.Utils.toCloseableIterator +import io.delta.kernel.statistics.DataFileStatistics +import io.delta.kernel.types.{GeometryType, StructType} +import io.delta.kernel.utils.CloseableIterable.inMemoryIterable +import io.delta.kernel.utils.DataFileStatus + +import org.scalatest.funsuite.AnyFunSuite + +/** + * End-to-end tests for data skipping with geometry column types using + * the StGeometryBoxesIntersect predicate. + * + * Tests bypass the Parquet writer by injecting DataFileStatistics with + * geometry min/max literals directly into the Delta log. + */ +class GeometryDataSkippingSuite extends AnyFunSuite with WriteUtils { + + // 4-quadrant layout: + // y=10 +----------+----------+ + // | NW file2 | NE file1 | + // y=7 + + + + // | x:[0,3] | x:[7,10] | + // y=3 +----------+----------+ + // | SW file0 | SE file3 | + // y=0 +----------+----------+ + // x=0 x=3 x=7 x=10 + private val fileExtents: Seq[(Double, Double, Double, Double)] = Seq( + (0.0, 0.0, 3.0, 3.0), // SW - file 0 + (7.0, 7.0, 10.0, 10.0), // NE - file 1 + (0.0, 7.0, 3.0, 10.0), // NW - file 2 + (7.0, 0.0, 10.0, 3.0) // SE - file 3 + ) + + private val colType = new GeometryType("OGC:CRS84") + + test("StGeometryBoxesIntersect data skipping on GeometryType column") { + withTempDirAndEngine { (tablePath, engine) => + val schema = new StructType().add("geom", colType) + writeFilesWithGeometryStats( + tablePath, + engine, + schema, + fileExtents) + + val snapshot = latestSnapshot(tablePath) + + def filesHit( + qMinX: Double, + qMinY: Double, + qMaxX: Double, + qMaxY: Double): Int = { + val pred = new StGeometryBoxesIntersect( + new Column("geom"), + Literal.ofGeospatialWKT(s"POINT ($qMinX $qMinY)", colType), + Literal.ofGeospatialWKT(s"POINT ($qMaxX $qMaxY)", colType)) + collectScanFileRows( + snapshot.getScanBuilder().withFilter(pred).build()).size + } + + // SW query - intersects only SW file + assert(filesHit(1.0, 1.0, 4.0, 4.0) == 1) + // NE query - intersects only NE file + assert(filesHit(8.0, 8.0, 11.0, 11.0) == 1) + // Center gap - intersects nothing + assert(filesHit(4.0, 4.0, 6.0, 6.0) == 0) + // West strip [1,1]-[4,9] - intersects SW and NW + assert(filesHit(1.0, 1.0, 4.0, 9.0) == 2) + // Top strip [0,8]-[11,11] - intersects NE and NW + assert(filesHit(0.0, 8.0, 11.0, 11.0) == 2) + // Global - intersects all 4 files + assert(filesHit(0.0, 0.0, 11.0, 11.0) == 4) + } + } + + test("StGeometryBoxesIntersect: file with null geometry stats is never skipped") { + withTempDirAndEngine { (tablePath, engine) => + val schema = new StructType().add("geom", colType) + val geomCol = new Column("geom") + + // File 0: SW quadrant with geometry stats + // File 1: no geometry stats -> must always be included + val extentsOpt: Seq[Option[(Double, Double, Double, Double)]] = + Seq(Some((0.0, 0.0, 3.0, 3.0)), None) + + extentsOpt.zipWithIndex.foreach { + case (extentOpt, idx) => + val txn = if (idx == 0) { + getCreateTxn(engine, tablePath, schema) + } else { + getUpdateTxn(engine, tablePath) + } + + val txnState = txn.getTransactionState(engine) + val writeContext = Transaction.getWriteContext( + engine, + txnState, + Collections.emptyMap()) + + val stats = extentOpt match { + case Some((minX, minY, maxX, maxY)) => + new DataFileStatistics( + 10, + Map(geomCol -> Literal.ofGeospatialWKT( + s"POINT ($minX $minY)", + colType)).asJava, + Map(geomCol -> Literal.ofGeospatialWKT( + s"POINT ($maxX $maxY)", + colType)).asJava, + Map(geomCol -> (0L: java.lang.Long)).asJava, + Optional.empty()) + case None => + new DataFileStatistics( + 10, + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyMap(), + Optional.empty()) + } + + val filePath = + engine.getFileSystemClient.resolvePath( + writeContext.getTargetDirectory + s"/part-$idx.parquet") + val fileStatus = new DataFileStatus( + filePath, + 1000, + 0L, + Optional.of(stats)) + + val actions = Transaction.generateAppendActions( + engine, + txnState, + toCloseableIterator(Seq(fileStatus).iterator.asJava), + writeContext) + commitTransaction( + txn, + engine, + inMemoryIterable(actions)) + } + + val snapshot = latestSnapshot(tablePath) + + def filesHit( + qMinX: Double, + qMinY: Double, + qMaxX: Double, + qMaxY: Double): Int = { + val pred = new StGeometryBoxesIntersect( + geomCol, + Literal.ofGeospatialWKT(s"POINT ($qMinX $qMinY)", colType), + Literal.ofGeospatialWKT(s"POINT ($qMaxX $qMaxY)", colType)) + collectScanFileRows( + snapshot.getScanBuilder().withFilter(pred).build()).size + } + + // NE query: SW file is provably non-overlapping, but + // null-stats file must be included + assert(filesHit(8.0, 8.0, 11.0, 11.0) == 1) + // SW query: SW file matches, null-stats file also included + assert(filesHit(1.0, 1.0, 4.0, 4.0) == 2) + } + } + + test("StGeometryBoxesIntersect combined with AND predicate on second column") { + withTempDirAndEngine { (tablePath, engine) => + val schema = new StructType() + .add("id", io.delta.kernel.types.IntegerType.INTEGER) + .add("geom", new GeometryType("OGC:CRS84")) + + val idCol = new Column("id") + val geomCol = new Column("geom") + + // File 0 (SW): id in [1,5], geom bbox [0,0]-[3,3] + // File 1 (NE): id in [10,20], geom bbox [7,7]-[10,10] + val files = Seq( + (1, 5, 0.0, 0.0, 3.0, 3.0), + (10, 20, 7.0, 7.0, 10.0, 10.0)) + + files.zipWithIndex.foreach { + case ((idMin, idMax, gMinX, gMinY, gMaxX, gMaxY), idx) => + val txn = if (idx == 0) { + getCreateTxn(engine, tablePath, schema) + } else { + getUpdateTxn(engine, tablePath) + } + + val txnState = txn.getTransactionState(engine) + val writeContext = Transaction.getWriteContext( + engine, + txnState, + Collections.emptyMap()) + + val stats = new DataFileStatistics( + 10, + Map( + idCol -> Literal.ofInt(idMin), + geomCol -> Literal.ofGeospatialWKT( + s"POINT ($gMinX $gMinY)", + colType)).asJava, + Map( + idCol -> Literal.ofInt(idMax), + geomCol -> Literal.ofGeospatialWKT( + s"POINT ($gMaxX $gMaxY)", + colType)).asJava, + Map( + idCol -> (0L: java.lang.Long), + geomCol -> (0L: java.lang.Long)).asJava, + Optional.empty()) + + val filePath = + engine.getFileSystemClient.resolvePath( + writeContext.getTargetDirectory + s"/part-$idx.parquet") + val fileStatus = new DataFileStatus( + filePath, + 1000, + 0L, + Optional.of(stats)) + + val actions = Transaction.generateAppendActions( + engine, + txnState, + toCloseableIterator(Seq(fileStatus).iterator.asJava), + writeContext) + commitTransaction( + txn, + engine, + inMemoryIterable(actions)) + } + + val snapshot = latestSnapshot(tablePath) + + // SW geometry + id <= 5: only File 0 survives both filters + val geoAndId = new io.delta.kernel.expressions.And( + new StGeometryBoxesIntersect( + geomCol, + Literal.ofGeospatialWKT("POINT (1.0 1.0)", colType), + Literal.ofGeospatialWKT("POINT (4.0 4.0)", colType)), + new io.delta.kernel.expressions.Predicate( + "<=", + idCol, + Literal.ofInt(5))) + + val files1 = collectScanFileRows( + snapshot.getScanBuilder().withFilter(geoAndId).build()) + assert(files1.size == 1) + + // NE geometry + id >= 15: only File 1 survives both filters + val geoAndId2 = new io.delta.kernel.expressions.And( + new StGeometryBoxesIntersect( + geomCol, + Literal.ofGeospatialWKT("POINT (8.0 8.0)", colType), + Literal.ofGeospatialWKT("POINT (11.0 11.0)", colType)), + new io.delta.kernel.expressions.Predicate( + ">=", + idCol, + Literal.ofInt(15))) + + val files2 = collectScanFileRows( + snapshot.getScanBuilder().withFilter(geoAndId2).build()) + assert(files2.size == 1) + + // Center geometry + any id: geometry miss skips both files + val geoCenterAny = new StGeometryBoxesIntersect( + geomCol, + Literal.ofGeospatialWKT("POINT (4.0 4.0)", colType), + Literal.ofGeospatialWKT("POINT (6.0 6.0)", colType)) + + val files3 = collectScanFileRows( + snapshot.getScanBuilder().withFilter(geoCenterAny).build()) + assert(files3.isEmpty) + } + } + + private def writeFilesWithGeometryStats( + tablePath: String, + engine: io.delta.kernel.engine.Engine, + schema: StructType, + extents: Seq[(Double, Double, Double, Double)]): Unit = { + val geomCol = new Column("geom") + extents.zipWithIndex.foreach { + case ((minX, minY, maxX, maxY), idx) => + val txn = if (idx == 0) { + getCreateTxn(engine, tablePath, schema) + } else { + getUpdateTxn(engine, tablePath) + } + + val txnState = txn.getTransactionState(engine) + val writeContext = Transaction.getWriteContext( + engine, + txnState, + Collections.emptyMap()) + + val stats = new DataFileStatistics( + 10, + Map(geomCol -> Literal.ofGeospatialWKT( + s"POINT ($minX $minY)", + colType)).asJava, + Map(geomCol -> Literal.ofGeospatialWKT( + s"POINT ($maxX $maxY)", + colType)).asJava, + Map(geomCol -> (0L: java.lang.Long)).asJava, + Optional.empty()) + + val filePath = + engine.getFileSystemClient.resolvePath( + writeContext.getTargetDirectory + s"/part-$idx.parquet") + val fileStatus = new DataFileStatus( + filePath, + 1000, + 0L, + Optional.of(stats)) + + val actions = Transaction.generateAppendActions( + engine, + txnState, + toCloseableIterator(Seq(fileStatus).iterator.asJava), + writeContext) + commitTransaction( + txn, + engine, + inMemoryIterable(actions)) + } + } +} diff --git a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/IcebergWriterCompatV1Suite.scala b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/IcebergWriterCompatV1Suite.scala index 02c64d0046b..a9e784ed477 100644 --- a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/IcebergWriterCompatV1Suite.scala +++ b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/IcebergWriterCompatV1Suite.scala @@ -470,7 +470,9 @@ trait IcebergWriterCompatV1SuiteBase testIncompatibleUnsupportedTableFeature( "changeDataFeed", - tablePropertiesToEnable = Map(TableConfig.CHANGE_DATA_FEED_ENABLED.getKey -> "true")) + tablePropertiesToEnable = Map(TableConfig.CHANGE_DATA_FEED_ENABLED.getKey -> "true"), + expectedErrorMessage = + "Table features [changeDataFeed] are incompatible with icebergWriterCompatV1") testIncompatibleUnsupportedTableFeature( "invariants", @@ -565,12 +567,6 @@ trait IcebergWriterCompatV1SuiteBase tablePropertiesToEnable = Map("delta.feature.defaultColumns" -> "supported"), expectedErrorMessage = "Unsupported Delta table feature") - // collations is not added to Kernel yet --> throws an error on feature lookup - testIncompatibleUnsupportedTableFeature( - "collations inactive", - tablePropertiesToEnable = Map("delta.feature.collations" -> "supported"), - expectedErrorMessage = "Unsupported Delta table feature") - /* ----- Legacy incompatible features allowed if they are inactive ----- */ test("legacy table features allowed with icebergWriterCompatV1 if inactive") { diff --git a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/engine/DefaultJsonHandlerSuite.scala b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/engine/DefaultJsonHandlerSuite.scala index a25a0187955..9b793f75d87 100644 --- a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/engine/DefaultJsonHandlerSuite.scala +++ b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/engine/DefaultJsonHandlerSuite.scala @@ -452,6 +452,63 @@ class DefaultJsonHandlerSuite extends AnyFunSuite with TestUtils with DefaultVec } } + test("parse geometry type as WKT string") { + val schema = new StructType() + .add("geom", GeometryType.ofDefault()) + testJsonParserWithSchema( + """{"geom": "POINT (1.0 2.0)"}""", + schema, + TestRow("POINT (1.0 2.0)")) + } + + test("parse geography type as WKT string") { + val schema = new StructType() + .add("geog", GeographyType.ofDefault()) + testJsonParserWithSchema( + """{"geog": "POINT (10.5 20.5)"}""", + schema, + TestRow("POINT (10.5 20.5)")) + } + + test("parse geometry/geography with null values") { + val schema = new StructType() + .add("geom", GeometryType.ofDefault(), true) + .add("geog", GeographyType.ofDefault(), true) + testJsonParserWithSchema( + """{"geom": null, "geog": null}""", + schema, + TestRow(null, null)) + } + + test("parse geometry POINT variants (Z, M, ZM)") { + val schema = new StructType() + .add("col1", GeometryType.ofDefault()) + testJsonParserWithSchema( + """{"col1": "POINT Z(1.0 2.0 3.0)"}""", + schema, + TestRow("POINT Z(1.0 2.0 3.0)")) + testJsonParserWithSchema( + """{"col1": "POINT M(1.0 2.0 4.0)"}""", + schema, + TestRow("POINT M(1.0 2.0 4.0)")) + testJsonParserWithSchema( + """{"col1": "POINT ZM(1.0 2.0 3.0 4.0)"}""", + schema, + TestRow("POINT ZM(1.0 2.0 3.0 4.0)")) + } + + test("parse geometry with non-string value throws") { + val schema = new StructType() + .add("geom", GeometryType.ofDefault()) + val e = intercept[RuntimeException] { + testJsonParserWithSchema( + """{"geom": 1234}""", + schema, + TestRow()) + } + assert(e.getMessage.contains("string")) + } + test("parse diverse type values in a map[string, string]") { val input = """ diff --git a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/utils/TestRow.scala b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/utils/TestRow.scala index 79facaead07..adfeed87a1a 100644 --- a/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/utils/TestRow.scala +++ b/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/utils/TestRow.scala @@ -110,6 +110,8 @@ object TestRow { case _: ArrayType => arrayValueToScalaSeq(row.getArray(i)) case _: MapType => mapValueToScalaMap(row.getMap(i)) case _: StructType => TestRow(row.getStruct(i)) + case _: GeometryType => row.getString(i) + case _: GeographyType => row.getString(i) case _ => throw new UnsupportedOperationException("unrecognized data type") } }.toSeq) diff --git a/sharing/src/main/scala/io/delta/sharing/spark/DeltaFormatSharingSource.scala b/sharing/src/main/scala/io/delta/sharing/spark/DeltaFormatSharingSource.scala index 4139dddd4ea..98942851420 100644 --- a/sharing/src/main/scala/io/delta/sharing/spark/DeltaFormatSharingSource.scala +++ b/sharing/src/main/scala/io/delta/sharing/spark/DeltaFormatSharingSource.scala @@ -36,7 +36,7 @@ import org.apache.spark.sql.delta.sources.{ } import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.JsonUtils -import io.delta.sharing.client.DeltaSharingClient +import io.delta.sharing.client.{DeltaSharingClient, DeltaSharingRestClient} import io.delta.sharing.client.util.ConfUtils import io.delta.sharing.client.model.{Table => DeltaSharingTable} @@ -409,7 +409,7 @@ case class DeltaFormatSharingSource( return null } - maybeGetLatestFileChangesFromServer(deltaSourceOffset) + maybeGetLatestFileChangesFromServer(deltaSourceOffset, wasConvertedFromLegacy) maybeMoveToNextVersion(deltaSource.latestOffset(startDeltaSourceOffsetOpt.orNull, limit)) } @@ -471,17 +471,43 @@ case class DeltaFormatSharingSource( * 2 usages: 1) used to compare with latestTableVersionInLocalDeltaLogOpt to * check whether new files are needed. 2) used for getTableFileChanges, * check more details in the function header. + * @param wasConvertedFromLegacy true when the startingOffset was converted from a legacy + * DeltaSharingSourceOffset checkpoint. */ - private def maybeGetLatestFileChangesFromServer(startingOffset: DeltaSourceOffset): Unit = { + private def maybeGetLatestFileChangesFromServer( + startingOffset: DeltaSourceOffset, + wasConvertedFromLegacy: Boolean): Unit = { // Use a local variable to avoid a difference in the two usages below. val latestTableVersion = getOrUpdateLatestTableVersion if (needNewFilesFromServer(startingOffset, latestTableVersion)) { - val endingVersionForQuery = + // Lucky case: offset conversion lands on a version + // boundary -- fetch file IDs with sha256 and pull up to + // maxVersionsPerRpc versions as a regular batch. + // + // Unlucky case: offset conversion is mid-version -- fetch + // file IDs with md5 and restrict to 1 version so the + // batch finishes the version and lands on a boundary. + val isLegacyMidVersion = wasConvertedFromLegacy && + startingOffset.index != DeltaSourceOffset.BASE_INDEX + val endingVersionForQuery = if (isLegacyMidVersion) { + startingOffset.reservoirVersion + } else { getEndingVersionForRpc(startingOffset, latestTableVersion) + } + + val fileIdHash: Option[String] = if (isLegacyMidVersion) { + Some(DeltaSharingRestClient.FILEIDHASH_MD5) + } else { + Some(DeltaSharingRestClient.FILEIDHASH_SHA256) + } + + logInfo(s"Legacy transition state: wasConvertedFromLegacy=$wasConvertedFromLegacy, " + + s"isLegacyMidVersion=$isLegacyMidVersion, fileIdHash=$fileIdHash, " + + s"endingVersionForQuery=$endingVersionForQuery") if (startingOffset.isInitialSnapshot || !options.readChangeFeed) { - getTableFileChanges(startingOffset, endingVersionForQuery) + getTableFileChanges(startingOffset, endingVersionForQuery, fileIdHash) } else { throw new UnsupportedOperationException("CDF Streaming is not supported yet.") } @@ -499,10 +525,12 @@ case class DeltaFormatSharingSource( * it will only load files since startingOffset.reservoirVersion. * @param endingVersionForQuery The ending version used for the query, always smaller than * the latest table version on server. + * @param fileIdHash Optional hash algorithm to use for file ID hashing. */ private def getTableFileChanges( startingOffset: DeltaSourceOffset, - endingVersionForQuery: Long): Unit = { + endingVersionForQuery: Long, + fileIdHash: Option[String] = None): Unit = { logInfo( s"Fetching files with table version(${startingOffset.reservoirVersion}), " + s"index(${startingOffset.index}), isInitialSnapshot(${startingOffset.isInitialSnapshot})," + @@ -520,7 +548,8 @@ case class DeltaFormatSharingSource( versionAsOf = Some(startingOffset.reservoirVersion), timestampAsOf = None, jsonPredicateHints = None, - refreshToken = None + refreshToken = None, + fileIdHash = fileIdHash ) val refreshFunc = DeltaSharingUtils.getRefresherForGetFiles( client = client, @@ -530,7 +559,8 @@ case class DeltaFormatSharingSource( versionAsOf = Some(startingOffset.reservoirVersion), timestampAsOf = None, jsonPredicateHints = None, - useRefreshToken = false + useRefreshToken = false, + fileIdHash = fileIdHash ) logInfo( s"Fetched ${tableFiles.lines.size} lines for table version ${tableFiles.version} from" + @@ -543,13 +573,15 @@ case class DeltaFormatSharingSource( val tableFiles = client.getFiles( table = table, startingVersion = startingOffset.reservoirVersion, - endingVersion = Some(endingVersionForQuery) + endingVersion = Some(endingVersionForQuery), + fileIdHash = fileIdHash ) val refreshFunc = DeltaSharingUtils.getRefresherForGetFilesWithStartingVersion( client = client, table = table, startingVersion = startingOffset.reservoirVersion, - endingVersion = Some(endingVersionForQuery) + endingVersion = Some(endingVersionForQuery), + fileIdHash = fileIdHash ) logInfo( s"Fetched ${tableFiles.lines.size} lines from startingVersion " + @@ -606,20 +638,27 @@ case class DeltaFormatSharingSource( // specifies the starting version. On restart, this Source // is instantiated, so we convert legacy offsets to // DeltaSourceOffset. - val endOffset = forceToDeltaSourceOffset(end)._1 + val (endOffset, endConvertedFromLegacy) = forceToDeltaSourceOffset(end) // When the query is past the initial snapshot, // startOffsetOption is defined and contains the starting // version. Convert from legacy offset if needed. - val startDeltaOffsetOption = - startOffsetOption.map(o => forceToDeltaSourceOffset(o)._1) + val (startDeltaOffsetOption, startConvertedFromLegacy) = + startOffsetOption.map(forceToDeltaSourceOffset) match { + case Some((offset, fromLegacy)) => (Some(offset), fromLegacy) + case None => (None, false) + } + // Possible legacy conversion scenarios: + // 1. start=None, end=legacy: initial snapshot restart. + // 2. start=legacy, end=new: post-snapshot, version + // boundary reached in one batch. + val wasConvertedFromLegacy = startConvertedFromLegacy || endConvertedFromLegacy val startingOffset = getStartingOffset(startDeltaOffsetOption, Some(endOffset)) - // Files should already be fetched in latestOffset; this - // is a safeguard in case files for startOffset are not - // present. Whether startOffset was converted from legacy - // does not matter here. - maybeGetLatestFileChangesFromServer(startingOffset = startingOffset) + // For most cases, files should already be fetched in latestOffset. When latestOffset + // succeeded but getBatch failed, on restart, Spark engine skips latestOffset, so we need to + // fetch files here. + maybeGetLatestFileChangesFromServer(startingOffset, wasConvertedFromLegacy) // Reset latestProcessedEndOffsetOption only when endOffset is larger. // Because with microbatch pipelining, we may get getBatch requests out of order. if (latestProcessedEndOffsetOption.isEmpty || diff --git a/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingCDFUtils.scala b/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingCDFUtils.scala index 7b8a8294c67..ae91cc4646d 100644 --- a/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingCDFUtils.scala +++ b/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingCDFUtils.scala @@ -52,8 +52,9 @@ object DeltaSharingCDFUtils extends Logging { // and also any metadata changes between [startingVersion, endingVersion], to put them in the // delta log. This is to allow delta library to check the metadata change and handle it // properly -- currently it throws error for column mapping changes. - val deltaTableFiles = - client.getCDFFiles(table, options.cdfOptions, includeHistoricalMetadata = true) + val deltaTableFiles = client.getCDFFiles( + table, options.cdfOptions, includeHistoricalMetadata = true, fileIdHash = None + ) logInfo( s"Fetched ${deltaTableFiles.lines.size} lines with cdf options ${options.cdfOptions} " + s"for table ${table} from delta sharing server, took ${getDuration(startTime)}s." diff --git a/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingDataSource.scala b/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingDataSource.scala index 59c6856ac11..5804b03f358 100644 --- a/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingDataSource.scala +++ b/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingDataSource.scala @@ -78,7 +78,8 @@ private[sharing] class DeltaSharingDataSource path, shareCredentialsOptions = options.shareCredentialsOptions, forStreaming = true, - responseFormat = DeltaSharingOptions.RESPONSE_FORMAT_PARQUET + responseFormat = DeltaSharingOptions.RESPONSE_FORMAT_PARQUET, + callerOrg = options.callerOrg ) val schemaToUse = deltaLog.snapshot().schema if (schemaToUse.isEmpty) { @@ -105,7 +106,8 @@ private[sharing] class DeltaSharingDataSource responseFormat = DeltaSharingOptions.RESPONSE_FORMAT_DELTA, // comma separated delta reader features, used to tell delta sharing server what delta // reader features the client is able to process. - readerFeatures = DeltaSharingUtils.STREAMING_SUPPORTED_READER_FEATURES.mkString(",") + readerFeatures = DeltaSharingUtils.STREAMING_SUPPORTED_READER_FEATURES.mkString(","), + callerOrg = options.callerOrg ) val dsTable = DeltaSharingTable( share = parsedPath.share, @@ -199,7 +201,8 @@ private[sharing] class DeltaSharingDataSource path, shareCredentialsOptions = options.shareCredentialsOptions, forStreaming = true, - responseFormat = DeltaSharingOptions.RESPONSE_FORMAT_PARQUET + responseFormat = DeltaSharingOptions.RESPONSE_FORMAT_PARQUET, + callerOrg = options.callerOrg ) DeltaSharingSource(SparkSession.active, deltaLog, options) } else if (responseFormat == DeltaSharingOptions.RESPONSE_FORMAT_DELTA) { @@ -217,7 +220,8 @@ private[sharing] class DeltaSharingDataSource responseFormat = DeltaSharingOptions.RESPONSE_FORMAT_DELTA, // comma separated delta reader features, used to tell delta sharing server what delta // reader features the client is able to process. - readerFeatures = DeltaSharingUtils.STREAMING_SUPPORTED_READER_FEATURES.mkString(",") + readerFeatures = DeltaSharingUtils.STREAMING_SUPPORTED_READER_FEATURES.mkString(","), + callerOrg = options.callerOrg ) val dsTable = DeltaSharingTable( share = parsedPath.share, @@ -273,7 +277,8 @@ private[sharing] class DeltaSharingDataSource shareCredentialsOptions = options.shareCredentialsOptions, forStreaming = true, versionAsOf = None, - timestampAsOf = None + timestampAsOf = None, + callerOrg = options.callerOrg ) logInfo(s"Streaming format resolved via getMetadata: ${deltaTableMetadata.respondedFormat} " + s"for path:$path") @@ -293,7 +298,8 @@ private[sharing] class DeltaSharingDataSource shareCredentialsOptions: Map[String, String], forStreaming: Boolean, versionAsOf: Option[Long], - timestampAsOf: Option[String]): (DeltaSharingTable, DeltaTableMetadata) = { + timestampAsOf: Option[String], + callerOrg: Option[String] = None): (DeltaSharingTable, DeltaTableMetadata) = { val responseFormat = { if (sqlContext.sparkSession.sessionState.conf.getConf( DeltaSQLConf.DELTA_SHARING_FORCE_DELTA_FORMAT)) { @@ -320,7 +326,8 @@ private[sharing] class DeltaSharingDataSource forStreaming = forStreaming, // Indicating that the client is able to process response format in both parquet and delta. responseFormat = responseFormat, - readerFeatures = readerFeatures + readerFeatures = readerFeatures, + callerOrg = callerOrg ) val dsTable = DeltaSharingTable( share = parsedPath.share, @@ -356,7 +363,8 @@ private[sharing] class DeltaSharingDataSource path, shareCredentialsOptions = options.shareCredentialsOptions, forStreaming = false, - responseFormat = options.responseFormat + responseFormat = options.responseFormat, + callerOrg = options.callerOrg ) deltaLog.createRelation( options.versionAsOf, @@ -377,7 +385,8 @@ private[sharing] class DeltaSharingDataSource responseFormat = options.responseFormat, // comma separated delta reader features, used to tell delta sharing server what delta // reader features the client is able to process. - readerFeatures = DeltaSharingUtils.SUPPORTED_READER_FEATURES.mkString(",") + readerFeatures = DeltaSharingUtils.SUPPORTED_READER_FEATURES.mkString(","), + callerOrg = options.callerOrg ) val dsTable = DeltaSharingTable( share = parsedPath.share, @@ -439,7 +448,8 @@ private[sharing] class DeltaSharingDataSource shareCredentialsOptions = options.shareCredentialsOptions, forStreaming = false, versionAsOf = options.versionAsOf, - timestampAsOf = options.timestampAsOf + timestampAsOf = options.timestampAsOf, + callerOrg = options.callerOrg ) if (deltaTableMetadata.respondedFormat == DeltaSharingOptions.RESPONSE_FORMAT_PARQUET) { @@ -450,7 +460,8 @@ private[sharing] class DeltaSharingDataSource options.shareCredentialsOptions, forStreaming = false, responseFormat = DeltaSharingOptions.RESPONSE_FORMAT_PARQUET, - initDeltaTableMetadata = Some(deltaTableMetadata) + initDeltaTableMetadata = Some(deltaTableMetadata), + callerOrg = options.callerOrg ) deltaLog.createRelation(options.versionAsOf, options.timestampAsOf, options.cdfOptions) } else if (deltaTableMetadata.respondedFormat == DeltaSharingOptions.RESPONSE_FORMAT_DELTA) { @@ -468,7 +479,8 @@ private[sharing] class DeltaSharingDataSource responseFormat = DeltaSharingOptions.RESPONSE_FORMAT_DELTA, // comma separated delta reader features, used to tell delta sharing server what delta // reader features the client is able to process. - readerFeatures = DeltaSharingUtils.SUPPORTED_READER_FEATURES.mkString(",") + readerFeatures = DeltaSharingUtils.SUPPORTED_READER_FEATURES.mkString(","), + callerOrg = options.callerOrg ) getHadoopFsRelationForDeltaSnapshotQuery( path = path, diff --git a/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingFileIndex.scala b/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingFileIndex.scala index c29dc2b805a..8d40daf58a6 100644 --- a/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingFileIndex.scala +++ b/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingFileIndex.scala @@ -149,7 +149,8 @@ case class DeltaSharingFileIndex( versionAsOf = params.options.versionAsOf, timestampAsOf = params.options.timestampAsOf, jsonPredicateHints = jsonPredicateHints, - refreshToken = None + refreshToken = None, + fileIdHash = None ) logInfo( s"Fetched ${deltaTableFiles.lines.size} lines for table $table with version " + diff --git a/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingUtils.scala b/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingUtils.scala index b27ea008763..ebd7999e54f 100644 --- a/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingUtils.scala +++ b/sharing/src/main/scala/io/delta/sharing/spark/DeltaSharingUtils.scala @@ -186,7 +186,8 @@ object DeltaSharingUtils extends Logging { versionAsOf: Option[Long], timestampAsOf: Option[String], jsonPredicateHints: Option[String], - useRefreshToken: Boolean): RefresherFunction = { refreshTokenOpt => + useRefreshToken: Boolean, + fileIdHash: Option[String] = None): RefresherFunction = { refreshTokenOpt => { // If versionAsOf is specified, ignore refresh token (e.g., in streaming queries) val tableFiles = client @@ -197,7 +198,8 @@ object DeltaSharingUtils extends Logging { versionAsOf = versionAsOf, timestampAsOf = timestampAsOf, jsonPredicateHints = jsonPredicateHints, - refreshToken = if (useRefreshToken) refreshTokenOpt else None + refreshToken = if (useRefreshToken) refreshTokenOpt else None, + fileIdHash = fileIdHash ) getTableRefreshResult(tableFiles) } @@ -212,12 +214,14 @@ object DeltaSharingUtils extends Logging { def getRefresherForGetCDFFiles( client: DeltaSharingClient, table: Table, - cdfOptions: Map[String, String]): RefresherFunction = { (_: Option[String]) => + cdfOptions: Map[String, String], + fileIdHash: Option[String] = None): RefresherFunction = { (_: Option[String]) => { val tableFiles = client.getCDFFiles( table = table, cdfOptions = cdfOptions, - includeHistoricalMetadata = true + includeHistoricalMetadata = true, + fileIdHash = fileIdHash ) getTableRefreshResult(tableFiles) } @@ -233,10 +237,16 @@ object DeltaSharingUtils extends Logging { client: DeltaSharingClient, table: Table, startingVersion: Long, - endingVersion: Option[Long]): RefresherFunction = { (_: Option[String]) => + endingVersion: Option[Long], + fileIdHash: Option[String] = None): RefresherFunction = { (_: Option[String]) => { val tableFiles = client - .getFiles(table = table, startingVersion = startingVersion, endingVersion = endingVersion) + .getFiles( + table = table, + startingVersion = startingVersion, + endingVersion = endingVersion, + fileIdHash = fileIdHash + ) getTableRefreshResult(tableFiles) } } diff --git a/sharing/src/test/scala-shims/spark-4.0/SharingStreamingTestShims.scala b/sharing/src/test/scala-shims/spark-4.0/SharingStreamingTestShims.scala index 790d4a51fea..2b67fb154ef 100644 --- a/sharing/src/test/scala-shims/spark-4.0/SharingStreamingTestShims.scala +++ b/sharing/src/test/scala-shims/spark-4.0/SharingStreamingTestShims.scala @@ -19,6 +19,7 @@ package io.delta.sharing.spark.test.shims import org.apache.spark.sql.execution.streaming.{ CheckpointFileManager => CheckpointFileManagerShim, CommitMetadata => CommitMetadataShim, + OffsetSeqLog => OffsetSeqLogShim, SerializedOffset => SerializedOffsetShim, StreamMetadata => StreamMetadataShim } @@ -33,6 +34,7 @@ object SharingStreamingTestShims { val CheckpointFileManager: CheckpointFileManagerShim.type = CheckpointFileManagerShim val CommitMetadata: CommitMetadataShim.type = CommitMetadataShim + type OffsetSeqLog = OffsetSeqLogShim val SerializedOffset: SerializedOffsetShim.type = SerializedOffsetShim val StreamMetadata: StreamMetadataShim.type = StreamMetadataShim diff --git a/sharing/src/test/scala-shims/spark-4.1/SharingStreamingTestShims.scala b/sharing/src/test/scala-shims/spark-4.1/SharingStreamingTestShims.scala index 8700b3b2977..eaa199abf81 100644 --- a/sharing/src/test/scala-shims/spark-4.1/SharingStreamingTestShims.scala +++ b/sharing/src/test/scala-shims/spark-4.1/SharingStreamingTestShims.scala @@ -18,7 +18,8 @@ package io.delta.sharing.spark.test.shims import org.apache.spark.sql.execution.streaming.checkpointing.{ CheckpointFileManager => CheckpointFileManagerShim, - CommitMetadata => CommitMetadataShim + CommitMetadata => CommitMetadataShim, + OffsetSeqLog => OffsetSeqLogShim } import org.apache.spark.sql.execution.streaming.runtime.{ SerializedOffset => SerializedOffsetShim, @@ -34,6 +35,7 @@ object SharingStreamingTestShims { val CheckpointFileManager: CheckpointFileManagerShim.type = CheckpointFileManagerShim val CommitMetadata: CommitMetadataShim.type = CommitMetadataShim + type OffsetSeqLog = OffsetSeqLogShim val SerializedOffset: SerializedOffsetShim.type = SerializedOffsetShim val StreamMetadata: StreamMetadataShim.type = StreamMetadataShim val StreamingCheckpointConstants: StreamingCheckpointConstantsShim.type = diff --git a/sharing/src/test/scala-shims/spark-4.2/SharingStreamingTestShims.scala b/sharing/src/test/scala-shims/spark-4.2/SharingStreamingTestShims.scala index 53057098891..ff6f23e9c73 100644 --- a/sharing/src/test/scala-shims/spark-4.2/SharingStreamingTestShims.scala +++ b/sharing/src/test/scala-shims/spark-4.2/SharingStreamingTestShims.scala @@ -18,7 +18,8 @@ package io.delta.sharing.spark.test.shims import org.apache.spark.sql.execution.streaming.checkpointing.{ CheckpointFileManager => CheckpointFileManagerShim, - CommitMetadata => CommitMetadataShim + CommitMetadata => CommitMetadataShim, + OffsetSeqLog => OffsetSeqLogShim } import org.apache.spark.sql.execution.streaming.runtime.{ SerializedOffset => SerializedOffsetShim, @@ -34,6 +35,7 @@ object SharingStreamingTestShims { val CheckpointFileManager: CheckpointFileManagerShim.type = CheckpointFileManagerShim val CommitMetadata: CommitMetadataShim.type = CommitMetadataShim + type OffsetSeqLog = OffsetSeqLogShim val SerializedOffset: SerializedOffsetShim.type = SerializedOffsetShim val StreamMetadata: StreamMetadataShim.type = StreamMetadataShim val StreamingCheckpointConstants: StreamingCheckpointConstantsShim.type = diff --git a/sharing/src/test/scala/io/delta/sharing/spark/DeltaFormatSharingSourceSuite.scala b/sharing/src/test/scala/io/delta/sharing/spark/DeltaFormatSharingSourceSuite.scala index 6cd9ac3add8..a99c6730513 100644 --- a/sharing/src/test/scala/io/delta/sharing/spark/DeltaFormatSharingSourceSuite.scala +++ b/sharing/src/test/scala/io/delta/sharing/spark/DeltaFormatSharingSourceSuite.scala @@ -37,6 +37,7 @@ import org.apache.spark.sql.delta.sharing.DeltaSharingTestSparkUtils import io.delta.sharing.spark.test.shims.SharingStreamingTestShims.{ CheckpointFileManager, CommitMetadata, + OffsetSeqLog, SerializedOffset, StreamingCheckpointConstants, StreamMetadata @@ -1742,4 +1743,468 @@ class DeltaFormatSharingSourceSuite } } } + + // E2E test: Legacy checkpoint at a version boundary (index=-1, i.e., the + // "lucky case") with isStartingVersion=true. The stream should transition + // fully to DeltaSourceOffset and use SHA256 file IDs. + test("E2E: legacy checkpoint at version boundary (lucky case) transitions cleanly") { + withTempDirs { (inputDir, outputDir, checkpointDir) => + val deltaTableName = "delta_table_lucky_case" + withTable(deltaTableName) { + sql(s"""CREATE TABLE $deltaTableName (value STRING) + |USING DELTA""".stripMargin) + sql(s"INSERT INTO $deltaTableName VALUES ('a'), ('b')") + sql(s"INSERT INTO $deltaTableName VALUES ('c'), ('d')") + val tableId = DeltaLog.forTable( + spark, new TableIdentifier(deltaTableName)) + .update().metadata.id + val sharedTableName = "shared_lucky_case" + val profileFile = prepareProfileFile(inputDir) + val tablePath = profileFile.getCanonicalPath + + s"#share1.default.$sharedTableName" + spark.sessionState.conf.setConfString( + "spark.delta.sharing.streaming.queryTableVersionIntervalSeconds", "10s") + + // Build checkpoint: legacy offset at version boundary (index=-1) + val checkpointPath = new Path(checkpointDir.getCanonicalPath) + // scalastyle:off deltahadoopconfiguration + val hadoopConf = spark.sessionState.newHadoopConf() + // scalastyle:on deltahadoopconfiguration + val fileManager = CheckpointFileManager.create(checkpointPath, hadoopConf) + val offsetsDir = StreamingCheckpointConstants.DIR_NAME_OFFSETS + val commitsDir = StreamingCheckpointConstants.DIR_NAME_COMMITS + val metaDir = StreamingCheckpointConstants.DIR_NAME_METADATA + fileManager.mkdirs(new Path(checkpointPath, offsetsDir)) + fileManager.mkdirs(new Path(checkpointPath, commitsDir)) + val metadataPath = new Path(checkpointPath, metaDir) + val streamId = java.util.UUID.randomUUID.toString + StreamMetadata.write(StreamMetadata(streamId), metadataPath, hadoopConf) + val offsetMetadataJson = + """{"batchWatermarkMs":0,"batchTimestampMs":0,"conf":{},"sourceMetadataInfo":{}}""" + + // Batch 0: version 1, at boundary (index=-1), initial snapshot + val legacyOffset0Json = + "{\"sourceVersion\":1," + + s""""tableId":"$tableId",""" + + "\"tableVersion\":1," + + "\"index\":-1," + + "\"isStartingVersion\":true}" + val offset0Content = + s"v1\n$offsetMetadataJson\n$legacyOffset0Json" + .getBytes(java.nio.charset.StandardCharsets.UTF_8) + val offset0Path = new Path(new Path(checkpointPath, offsetsDir), "0") + val offset0Out = fileManager.createAtomic(offset0Path, overwriteIfPossible = true) + offset0Out.write(offset0Content) + offset0Out.close() + val commit0Content = s"v1\n${CommitMetadata(0).json}" + .getBytes(java.nio.charset.StandardCharsets.UTF_8) + val commit0Path = new Path(new Path(checkpointPath, commitsDir), "0") + val commit0Out = fileManager.createAtomic(commit0Path, overwriteIfPossible = true) + commit0Out.write(commit0Content) + commit0Out.close() + + val autoResolveKey = DeltaSQLConf + .DELTA_SHARING_STREAMING_AUTO_RESOLVE_RESPONSE_FORMAT.key + withSQLConf( + (getDeltaSharingClassesSQLConf ++ Seq( + autoResolveKey -> "true" + )).toSeq: _* + ) { + prepareMockedClientMetadata(deltaTableName, sharedTableName) + // getBatch(None, offset_0) with isInitialSnapshot=true uses + // snapshot at the same version. + prepareMockedClientAndFileSystemResult( + deltaTableName, sharedTableName, versionAsOf = Some(1L)) + // Since the legacy offset is at a version boundary (index=-1 -> BASE_INDEX), + // this is the "lucky case": the stream should use SHA256 file IDs and + // fetch multiple versions normally. + prepareMockedClientAndFileSystemResultForStreaming( + deltaTableName, sharedTableName, 1L, 2L) + prepareMockedClientAndFileSystemResultForStreaming( + deltaTableName, sharedTableName, 2L, 2L) + prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) + + val q = spark.readStream + .format("deltaSharing") + .option("responseFormat", "delta") + .load(tablePath) + .writeStream + .format("delta") + .option("checkpointLocation", checkpointDir.toString) + .start(outputDir.toString) + try { + q.processAllAvailable() + } finally { + q.stop() + } + // Replayed batch 0 produces v1 snapshot, then version 2 + checkAnswer( + spark.read.format("delta").load(outputDir.getCanonicalPath), + Seq("a", "b", "c", "d").toDF()) + + // Validate the final offset is in DeltaSourceOffset format + // (has reservoirVersion, not tableVersion). + val offsetLog = new OffsetSeqLog( + spark, s"${checkpointDir.getCanonicalPath}/offsets") + val (latestBatchId, latestOffsetSeq) = offsetLog.getLatest().get + val offsetJson = latestOffsetSeq.offsets.head.get.json() + assert(offsetJson.contains("reservoirVersion"), + s"Expected DeltaSourceOffset (reservoirVersion) but got: $offsetJson") + assert(!offsetJson.contains("tableVersion"), + s"Expected no legacy tableVersion in final offset but got: $offsetJson") + } + } + } + } + + // E2E: Legacy checkpoint mid-version (index != -1, the "unlucky case") + // with isStartingVersion=false. The stream should restrict to 1 version + // with MD5 file IDs, then transition after reaching a boundary. + test("E2E: legacy checkpoint mid-version (unlucky case) " + + "finishes version before transitioning") { + withTempDirs { (inputDir, outputDir, checkpointDir) => + val deltaTableName = "delta_table_unlucky_case" + withTable(deltaTableName) { + sql(s"""CREATE TABLE $deltaTableName (value STRING) + |USING DELTA""".stripMargin) + sql(s"INSERT INTO $deltaTableName VALUES ('v1a'), ('v1b')") + // Version 2: exactly 2 files (1 row each) for the mid-version case. + // parallelize with numSlices=2 places each row in its own partition. + spark.createDataFrame( + spark.sparkContext.parallelize(Seq(Row("v2a"), Row("v2b")), 2), + new StructType().add("value", StringType) + ).write.mode("append").insertInto(deltaTableName) + sql(s"INSERT INTO $deltaTableName VALUES ('v3a'), ('v3b')") + val tableId = DeltaLog.forTable( + spark, new TableIdentifier(deltaTableName)) + .update().metadata.id + val sharedTableName = "shared_unlucky_case" + val profileFile = prepareProfileFile(inputDir) + val tablePath = profileFile.getCanonicalPath + + s"#share1.default.$sharedTableName" + spark.sessionState.conf.setConfString( + "spark.delta.sharing.streaming.queryTableVersionIntervalSeconds", "10s") + + // Build checkpoint: legacy offset mid-version (index=0, not -1) + val checkpointPath = new Path(checkpointDir.getCanonicalPath) + // scalastyle:off deltahadoopconfiguration + val hadoopConf = spark.sessionState.newHadoopConf() + // scalastyle:on deltahadoopconfiguration + val fileManager = CheckpointFileManager.create(checkpointPath, hadoopConf) + val offsetsDir = StreamingCheckpointConstants.DIR_NAME_OFFSETS + val commitsDir = StreamingCheckpointConstants.DIR_NAME_COMMITS + val metaDir = StreamingCheckpointConstants.DIR_NAME_METADATA + fileManager.mkdirs(new Path(checkpointPath, offsetsDir)) + fileManager.mkdirs(new Path(checkpointPath, commitsDir)) + val metadataPath = new Path(checkpointPath, metaDir) + val streamId = java.util.UUID.randomUUID.toString + StreamMetadata.write(StreamMetadata(streamId), metadataPath, hadoopConf) + val offsetMetadataJson = + """{"batchWatermarkMs":0,"batchTimestampMs":0,"conf":{},"sourceMetadataInfo":{}}""" + + // Batch 0: mid-version at version 2, index 0 (processed 1 of 2 files) + val legacyOffset0Json = + "{\"sourceVersion\":1," + + s""""tableId":"$tableId",""" + + "\"tableVersion\":2," + + "\"index\":0," + + "\"isStartingVersion\":false}" + val offset0Content = + s"v1\n$offsetMetadataJson\n$legacyOffset0Json" + .getBytes(java.nio.charset.StandardCharsets.UTF_8) + val offset0Path = new Path(new Path(checkpointPath, offsetsDir), "0") + val offset0Out = fileManager.createAtomic(offset0Path, overwriteIfPossible = true) + offset0Out.write(offset0Content) + offset0Out.close() + val commit0Content = s"v1\n${CommitMetadata(0).json}" + .getBytes(java.nio.charset.StandardCharsets.UTF_8) + val commit0Path = new Path(new Path(checkpointPath, commitsDir), "0") + val commit0Out = fileManager.createAtomic(commit0Path, overwriteIfPossible = true) + commit0Out.write(commit0Content) + commit0Out.close() + + val autoResolveKey = DeltaSQLConf + .DELTA_SHARING_STREAMING_AUTO_RESOLVE_RESPONSE_FORMAT.key + withSQLConf( + (getDeltaSharingClassesSQLConf ++ Seq( + autoResolveKey -> "true" + )).toSeq: _* + ) { + prepareMockedClientMetadata(deltaTableName, sharedTableName) + // getBatch(None, offset_0) computes starting offset at + // version 1 as initial snapshot, so we need this mock. + prepareMockedClientAndFileSystemResult( + deltaTableName, sharedTableName, versionAsOf = Some(1L)) + // For mid-version case, the stream should restrict to version 2 + // only and use MD5 file IDs. + prepareMockedClientAndFileSystemResultForStreaming( + deltaTableName, sharedTableName, 2L, 2L) + // After transitioning at version boundary, stream fetches + // remaining versions normally. + prepareMockedClientAndFileSystemResultForStreaming( + deltaTableName, sharedTableName, 2L, 3L) + prepareMockedClientAndFileSystemResultForStreaming( + deltaTableName, sharedTableName, 3L, 3L) + prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) + + val q = spark.readStream + .format("deltaSharing") + .option("responseFormat", "delta") + .load(tablePath) + .writeStream + .format("delta") + .option("checkpointLocation", checkpointDir.toString) + .start(outputDir.toString) + try { + q.processAllAvailable() + } finally { + q.stop() + } + // The legacy checkpoint was at version 2, index 0 (1 of 2 files + // processed). The stream finishes the remaining file in version 2, + // transitions to DeltaSourceOffset at the version boundary, then + // processes version 3 normally. + val outputValues = spark.read.format("delta") + .load(outputDir.getCanonicalPath) + .collect().map(_.getString(0)).toSet + assert(outputValues.size == 3, + s"Expected 3 rows (1 remaining v2 + 2 v3) but got: $outputValues") + assert(outputValues.contains("v3a") && outputValues.contains("v3b"), + s"Expected v3a and v3b in output but got: $outputValues") + assert(outputValues.intersect(Set("v2a", "v2b")).size == 1, + s"Expected exactly one v2 row from the unprocessed file but got: $outputValues") + + // Validate the final offset has transitioned to DeltaSourceOffset format. + val offsetLog = new OffsetSeqLog( + spark, s"${checkpointDir.getCanonicalPath}/offsets") + val (latestBatchId, latestOffsetSeq) = offsetLog.getLatest().get + val offsetJson = latestOffsetSeq.offsets.head.get.json() + assert(offsetJson.contains("reservoirVersion"), + s"Expected DeltaSourceOffset (reservoirVersion) but got: $offsetJson") + assert(!offsetJson.contains("tableVersion"), + s"Expected no legacy tableVersion in final offset but got: $offsetJson") + } + } + } + } + + // Test that fileIdHash is correctly passed to the server: SHA256 for + // version-boundary legacy offsets (lucky case), MD5 for mid-version + // legacy offsets (unlucky case), and SHA256 after transition. + test("fileIdHash: SHA256 for boundary legacy offset") { + withTempDirs { (inputDir, outputDir, checkpointDir) => + val deltaTableName = "delta_table_fileidhash" + withTable(deltaTableName) { + sql(s"""CREATE TABLE $deltaTableName (value STRING) + |USING DELTA""".stripMargin) + sql(s"INSERT INTO $deltaTableName VALUES ('a'), ('b')") + sql(s"INSERT INTO $deltaTableName VALUES ('c'), ('d')") + val tableId = DeltaLog.forTable( + spark, new TableIdentifier(deltaTableName)) + .update().metadata.id + val sharedTableName = "shared_fileidhash" + val profileFile = prepareProfileFile(inputDir) + val tablePath = profileFile.getCanonicalPath + + s"#share1.default.$sharedTableName" + spark.sessionState.conf.setConfString( + "spark.delta.sharing.streaming.queryTableVersionIntervalSeconds", "10s") + + val checkpointPath = new Path(checkpointDir.getCanonicalPath) + // scalastyle:off deltahadoopconfiguration + val hadoopConf = spark.sessionState.newHadoopConf() + // scalastyle:on deltahadoopconfiguration + val fileManager = CheckpointFileManager.create(checkpointPath, hadoopConf) + val offsetsDir = StreamingCheckpointConstants.DIR_NAME_OFFSETS + val commitsDir = StreamingCheckpointConstants.DIR_NAME_COMMITS + val metaDir = StreamingCheckpointConstants.DIR_NAME_METADATA + fileManager.mkdirs(new Path(checkpointPath, offsetsDir)) + fileManager.mkdirs(new Path(checkpointPath, commitsDir)) + val metadataPath = new Path(checkpointPath, metaDir) + val streamId = java.util.UUID.randomUUID.toString + StreamMetadata.write(StreamMetadata(streamId), metadataPath, hadoopConf) + val offsetMetadataJson = + """{"batchWatermarkMs":0,"batchTimestampMs":0,"conf":{},"sourceMetadataInfo":{}}""" + + // Version boundary (lucky case) should use SHA256 + val boundaryOffsetJson = + "{\"sourceVersion\":1," + + s""""tableId":"$tableId",""" + + "\"tableVersion\":1," + + "\"index\":-1," + + "\"isStartingVersion\":true}" + val offset0Content = + s"v1\n$offsetMetadataJson\n$boundaryOffsetJson" + .getBytes(java.nio.charset.StandardCharsets.UTF_8) + val offset0Path = new Path(new Path(checkpointPath, offsetsDir), "0") + val offset0Out = fileManager.createAtomic(offset0Path, overwriteIfPossible = true) + offset0Out.write(offset0Content) + offset0Out.close() + val commit0Content = s"v1\n${CommitMetadata(0).json}" + .getBytes(java.nio.charset.StandardCharsets.UTF_8) + val commit0Path = new Path(new Path(checkpointPath, commitsDir), "0") + val commit0Out = fileManager.createAtomic(commit0Path, overwriteIfPossible = true) + commit0Out.write(commit0Content) + commit0Out.close() + + val autoResolveKey = DeltaSQLConf + .DELTA_SHARING_STREAMING_AUTO_RESOLVE_RESPONSE_FORMAT.key + withSQLConf( + (getDeltaSharingClassesSQLConf ++ Seq( + autoResolveKey -> "true" + )).toSeq: _* + ) { + prepareMockedClientMetadata(deltaTableName, sharedTableName) + prepareMockedClientAndFileSystemResult( + deltaTableName, sharedTableName, versionAsOf = Some(1L)) + prepareMockedClientAndFileSystemResultForStreaming( + deltaTableName, sharedTableName, 1L, 2L) + prepareMockedClientAndFileSystemResultForStreaming( + deltaTableName, sharedTableName, 2L, 2L) + prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) + + TestClientForDeltaFormatSharing.clearFileIdHashHistory() + + val q = spark.readStream + .format("deltaSharing") + .option("responseFormat", "delta") + .load(tablePath) + .writeStream + .format("delta") + .option("checkpointLocation", checkpointDir.toString) + .start(outputDir.toString) + try { + q.processAllAvailable() + } finally { + q.stop() + } + + // All streaming getFiles calls should use SHA256 for the + // lucky case (version boundary). + val history = TestClientForDeltaFormatSharing.getFileIdHashHistory + val streamingCalls = history.filter { case (name, qt, _) => + name == sharedTableName && qt.startsWith("getFiles_streaming") + } + assert(streamingCalls.nonEmpty, "Expected at least one streaming getFiles call") + streamingCalls.foreach { case (_, queryType, fileIdHash) => + assert(fileIdHash.contains(DeltaSharingRestClient.FILEIDHASH_SHA256), + s"Expected SHA256 for boundary legacy offset but got $fileIdHash in $queryType") + } + } + } + } + } + + test("fileIdHash: MD5 for mid-version legacy offset") { + withTempDirs { (inputDir, outputDir, checkpointDir) => + val deltaTableName = "delta_table_fileidhash_mid" + withTable(deltaTableName) { + sql(s"""CREATE TABLE $deltaTableName (value STRING) + |USING DELTA""".stripMargin) + sql(s"INSERT INTO $deltaTableName VALUES ('e'), ('f')") + sql(s"INSERT INTO $deltaTableName VALUES ('g'), ('h')") + sql(s"INSERT INTO $deltaTableName VALUES ('i'), ('j')") + val tableId = DeltaLog.forTable( + spark, new TableIdentifier(deltaTableName)) + .update().metadata.id + val sharedTableName = "shared_fileidhash_mid" + val profileFile = prepareProfileFile(inputDir) + val tablePath = profileFile.getCanonicalPath + + s"#share1.default.$sharedTableName" + spark.sessionState.conf.setConfString( + "spark.delta.sharing.streaming.queryTableVersionIntervalSeconds", "10s") + + val checkpointPath = new Path(checkpointDir.getCanonicalPath) + // scalastyle:off deltahadoopconfiguration + val hadoopConf = spark.sessionState.newHadoopConf() + // scalastyle:on deltahadoopconfiguration + val fileManager = CheckpointFileManager.create(checkpointPath, hadoopConf) + val offsetsDir = StreamingCheckpointConstants.DIR_NAME_OFFSETS + val commitsDir = StreamingCheckpointConstants.DIR_NAME_COMMITS + val metaDir = StreamingCheckpointConstants.DIR_NAME_METADATA + fileManager.mkdirs(new Path(checkpointPath, offsetsDir)) + fileManager.mkdirs(new Path(checkpointPath, commitsDir)) + val metadataPath = new Path(checkpointPath, metaDir) + val streamId = java.util.UUID.randomUUID.toString + StreamMetadata.write(StreamMetadata(streamId), metadataPath, hadoopConf) + val offsetMetadataJson = + """{"batchWatermarkMs":0,"batchTimestampMs":0,"conf":{},"sourceMetadataInfo":{}}""" + + // Mid-version offset: version 2, index 0 (not at boundary) + val midVersionOffsetJson = + "{\"sourceVersion\":1," + + s""""tableId":"$tableId",""" + + "\"tableVersion\":2," + + "\"index\":0," + + "\"isStartingVersion\":false}" + val offset0Content = + s"v1\n$offsetMetadataJson\n$midVersionOffsetJson" + .getBytes(java.nio.charset.StandardCharsets.UTF_8) + val offset0Path = new Path(new Path(checkpointPath, offsetsDir), "0") + val offset0Out = fileManager.createAtomic(offset0Path, overwriteIfPossible = true) + offset0Out.write(offset0Content) + offset0Out.close() + val commit0Content = s"v1\n${CommitMetadata(0).json}" + .getBytes(java.nio.charset.StandardCharsets.UTF_8) + val commit0Path = new Path(new Path(checkpointPath, commitsDir), "0") + val commit0Out = fileManager.createAtomic(commit0Path, overwriteIfPossible = true) + commit0Out.write(commit0Content) + commit0Out.close() + + val autoResolveKey = DeltaSQLConf + .DELTA_SHARING_STREAMING_AUTO_RESOLVE_RESPONSE_FORMAT.key + withSQLConf( + (getDeltaSharingClassesSQLConf ++ Seq( + autoResolveKey -> "true" + )).toSeq: _* + ) { + prepareMockedClientMetadata(deltaTableName, sharedTableName) + // getBatch recovery loads snapshot at version 2-1=1 + prepareMockedClientAndFileSystemResult( + deltaTableName, sharedTableName, versionAsOf = Some(1L)) + // Mid-version: restricted to version 2 only (MD5) + prepareMockedClientAndFileSystemResultForStreaming( + deltaTableName, sharedTableName, 2L, 2L) + // After transition: fetches from version 2 onwards normally (SHA256) + prepareMockedClientAndFileSystemResultForStreaming( + deltaTableName, sharedTableName, 2L, 3L) + prepareMockedClientAndFileSystemResultForStreaming( + deltaTableName, sharedTableName, 3L, 3L) + prepareMockedClientGetTableVersion(deltaTableName, sharedTableName) + + TestClientForDeltaFormatSharing.clearFileIdHashHistory() + + val q = spark.readStream + .format("deltaSharing") + .option("responseFormat", "delta") + .load(tablePath) + .writeStream + .format("delta") + .option("checkpointLocation", checkpointDir.toString) + .start(outputDir.toString) + try { + q.processAllAvailable() + } finally { + q.stop() + } + + val history = TestClientForDeltaFormatSharing.getFileIdHashHistory + val streamingCalls = history.filter { case (name, qt, _) => + name == sharedTableName && qt.startsWith("getFiles_streaming") + } + assert(streamingCalls.nonEmpty, "Expected at least one streaming getFiles call") + // The first streaming call should use MD5 (mid-version legacy), + // subsequent calls after transition should use SHA256. + val firstCall = streamingCalls.head + assert(firstCall._3.contains(DeltaSharingRestClient.FILEIDHASH_MD5), + s"Expected MD5 for mid-version legacy offset but got ${firstCall._3}") + if (streamingCalls.size > 1) { + streamingCalls.tail.foreach { case (_, queryType, fileIdHash) => + assert(fileIdHash.contains(DeltaSharingRestClient.FILEIDHASH_SHA256), + s"Expected SHA256 after transition but got $fileIdHash in $queryType") + } + } + } + } + } + } } diff --git a/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingCDFUtilsSuite.scala b/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingCDFUtilsSuite.scala index 8fe6319a33b..11f7752d55c 100644 --- a/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingCDFUtilsSuite.scala +++ b/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingCDFUtilsSuite.scala @@ -88,7 +88,9 @@ class TestDeltaSharingClientForCDFUtils( asyncQueryMaxDuration: Long = 600000L, tokenExchangeMaxRetries: Int = 5, tokenExchangeMaxRetryDurationInSeconds: Int = 60, - tokenRenewalThresholdInSeconds: Int = 600) + tokenRenewalThresholdInSeconds: Int = 600, + callerOrg: String = "", + skipFileIdHashVerification: Boolean = false) extends DeltaSharingClient { import CDFTesTUtils._ @@ -121,7 +123,8 @@ class TestDeltaSharingClientForCDFUtils( versionAsOf: Option[Long], timestampAsOf: Option[String], jsonPredicateHints: Option[String], - refreshToken: Option[String] + refreshToken: Option[String], + fileIdHash: Option[String] ): DeltaTableFiles = { throw new UnsupportedOperationException("getFiles is not supported now.") } @@ -129,7 +132,8 @@ class TestDeltaSharingClientForCDFUtils( override def getFiles( table: Table, startingVersion: Long, - endingVersion: Option[Long] + endingVersion: Option[Long], + fileIdHash: Option[String] ): DeltaTableFiles = { throw new UnsupportedOperationException(s"getFiles with startingVersion($startingVersion)") } @@ -137,7 +141,8 @@ class TestDeltaSharingClientForCDFUtils( override def getCDFFiles( table: Table, cdfOptions: Map[String, String], - includeHistoricalMetadata: Boolean + includeHistoricalMetadata: Boolean, + fileIdHash: Option[String] ): DeltaTableFiles = { numGetFileCalls += 1 DeltaTableFiles( diff --git a/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingDataSourceDeltaSuite.scala b/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingDataSourceDeltaSuite.scala index f831130b361..314a9773686 100644 --- a/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingDataSourceDeltaSuite.scala +++ b/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingDataSourceDeltaSuite.scala @@ -1696,6 +1696,53 @@ trait DeltaSharingDataSourceDeltaSuiteBase } } } + test("callerOrg option is passed to DeltaSharingRestClient") { + withTempDirs { (inputDir, outputDir, checkpointDir) => + val deltaTableName = "delta_table_caller_org" + withTable(deltaTableName) { + createSimpleTable(deltaTableName, enableCdf = false) + sql(s"""INSERT INTO $deltaTableName VALUES (1, "one")""") + + val sharedTableName = "shared_table_caller_org" + prepareMockedClientAndFileSystemResult( + deltaTableName, sharedTableName) + DeltaSharingUtils.overrideSingleBlock[Long]( + blockId = TestClientForDeltaFormatSharing.getBlockId( + sharedTableName, "getTableVersion"), + value = 1 + ) + + withSQLConf(getDeltaSharingClassesSQLConf.toSeq: _*) { + val profileFile = prepareProfileFile(inputDir) + val tablePath = + s"${profileFile.getCanonicalPath}#share1.default.$sharedTableName" + + TestClientForDeltaFormatSharing.lastCallerOrg = "" + spark.read + .format("deltaSharing") + .option("responseFormat", "delta") + .option(DeltaSharingOptions.CALLER_ORG_OPTION, "test-org") + .load(tablePath) + .collect() + assert( + TestClientForDeltaFormatSharing.lastCallerOrg == "test-org", + "callerOrg should be passed through to the client" + ) + + TestClientForDeltaFormatSharing.lastCallerOrg = "" + spark.read + .format("deltaSharing") + .option("responseFormat", "delta") + .load(tablePath) + .collect() + assert( + TestClientForDeltaFormatSharing.lastCallerOrg == "", + "callerOrg should be empty when not set" + ) + } + } + } + } } class DeltaSharingDataSourceDeltaSuite extends DeltaSharingDataSourceDeltaSuiteBase {} diff --git a/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingFileIndexSuite.scala b/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingFileIndexSuite.scala index 4166444f2d0..080862487cb 100644 --- a/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingFileIndexSuite.scala +++ b/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingFileIndexSuite.scala @@ -100,7 +100,9 @@ class TestDeltaSharingClientForFileIndex( asyncQueryMaxDuration: Long = 600000L, tokenExchangeMaxRetries: Int = 5, tokenExchangeMaxRetryDurationInSeconds: Int = 60, - tokenRenewalThresholdInSeconds: Int = 600) + tokenRenewalThresholdInSeconds: Int = 600, + callerOrg: String = "", + skipFileIdHashVerification: Boolean = false) extends DeltaSharingClient { import TestUtils._ @@ -148,7 +150,8 @@ class TestDeltaSharingClientForFileIndex( versionAsOf: Option[Long], timestampAsOf: Option[String], jsonPredicateHints: Option[String], - refreshToken: Option[String] + refreshToken: Option[String], + fileIdHash: Option[String] ): DeltaTableFiles = { numGetFileCalls += 1 limit.foreach(lim => savedLimits = savedLimits :+ lim) @@ -175,7 +178,8 @@ class TestDeltaSharingClientForFileIndex( override def getFiles( table: Table, startingVersion: Long, - endingVersion: Option[Long] + endingVersion: Option[Long], + fileIdHash: Option[String] ): DeltaTableFiles = { throw new UnsupportedOperationException(s"getFiles with startingVersion($startingVersion)") } @@ -183,7 +187,8 @@ class TestDeltaSharingClientForFileIndex( override def getCDFFiles( table: Table, cdfOptions: Map[String, String], - includeHistoricalMetadata: Boolean + includeHistoricalMetadata: Boolean, + fileIdHash: Option[String] ): DeltaTableFiles = { throw new UnsupportedOperationException( s"getCDFFiles with cdfOptions:[$cdfOptions], " + diff --git a/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingUtilsSuite.scala b/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingUtilsSuite.scala index ed9dcd2538e..3e682086969 100644 --- a/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingUtilsSuite.scala +++ b/sharing/src/test/scala/io/delta/sharing/spark/DeltaSharingUtilsSuite.scala @@ -133,7 +133,8 @@ class DeltaSharingUtilsSuite extends SparkFunSuite with SharedSparkContext { versionAsOf: Option[Long], timestampAsOf: Option[String], jsonPredicateHints: Option[String], - refreshToken: Option[String] + refreshToken: Option[String], + fileIdHash: Option[String] ): DeltaTableFiles = { val file = getAddFileStr() val dv = getDeletionVectorStr() @@ -144,8 +145,12 @@ class DeltaSharingUtilsSuite extends SparkFunSuite with SharedSparkContext { ) } - override def getFiles(table: Table, startingVersion: Long, endingVersion: Option[Long]) - : DeltaTableFiles = { + override def getFiles( + table: Table, + startingVersion: Long, + endingVersion: Option[Long], + fileIdHash: Option[String] + ): DeltaTableFiles = { val file = getAddFileStr() val dv = getDeletionVectorStr() DeltaTableFiles( @@ -158,7 +163,8 @@ class DeltaSharingUtilsSuite extends SparkFunSuite with SharedSparkContext { override def getCDFFiles( table: Table, cdfOptions: Map[String, String], - includeHistoricalMetadata: Boolean): DeltaTableFiles = { + includeHistoricalMetadata: Boolean, + fileIdHash: Option[String]): DeltaTableFiles = { val file = getAddFileStr() val dv = getDeletionVectorStr() val cdc = getCdcStr() @@ -283,11 +289,12 @@ class DeltaSharingUtilsSuite extends SparkFunSuite with SharedSparkContext { versionAsOf: Option[Long], timestampAsOf: Option[String], jsonPredicateHints: Option[String], - refreshToken: Option[String] + refreshToken: Option[String], + fileIdHash: Option[String] ): DeltaTableFiles = { lastRefreshToken = refreshToken super.getFiles(table, predicates, limit, versionAsOf, timestampAsOf, - jsonPredicateHints, refreshToken) + jsonPredicateHints, refreshToken, fileIdHash) } } diff --git a/sharing/src/test/scala/io/delta/sharing/spark/TestClientForDeltaFormatSharing.scala b/sharing/src/test/scala/io/delta/sharing/spark/TestClientForDeltaFormatSharing.scala index c8828f2c25b..d944cbf091e 100644 --- a/sharing/src/test/scala/io/delta/sharing/spark/TestClientForDeltaFormatSharing.scala +++ b/sharing/src/test/scala/io/delta/sharing/spark/TestClientForDeltaFormatSharing.scala @@ -60,7 +60,9 @@ private[spark] class TestClientForDeltaFormatSharing( asyncQueryMaxDuration: Long = 600000L, tokenExchangeMaxRetries: Int = 5, tokenExchangeMaxRetryDurationInSeconds: Int = 60, - tokenRenewalThresholdInSeconds: Int = 600) + tokenRenewalThresholdInSeconds: Int = 600, + callerOrg: String = "", + skipFileIdHashVerification: Boolean = false) extends DeltaSharingClient { private val supportedReaderFeatures: Seq[String] = Seq( @@ -83,6 +85,8 @@ private[spark] class TestClientForDeltaFormatSharing( import TestClientForDeltaFormatSharing._ + TestClientForDeltaFormatSharing.lastCallerOrg = callerOrg + override def listAllTables(): Seq[Table] = throw new UnsupportedOperationException("not needed") override def getMetadata( @@ -144,11 +148,16 @@ private[spark] class TestClientForDeltaFormatSharing( versionAsOf: Option[Long], timestampAsOf: Option[String], jsonPredicateHints: Option[String], - refreshToken: Option[String] + refreshToken: Option[String], + fileIdHash: Option[String] ): DeltaTableFiles = { val tableFullName = s"${table.share}.${table.schema}.${table.name}" limit.foreach(lim => TestClientForDeltaFormatSharing.limits.put(tableFullName, lim)) TestClientForDeltaFormatSharing.requestedFormat.put(tableFullName, responseFormat) + TestClientForDeltaFormatSharing.fileIdHashHistory.synchronized { + TestClientForDeltaFormatSharing.fileIdHashHistory += + ((table.name, "getFiles_snapshot", fileIdHash)) + } jsonPredicateHints.foreach(p => TestClientForDeltaFormatSharing.jsonPredicateHints.put(tableFullName, p)) @@ -206,7 +215,8 @@ private[spark] class TestClientForDeltaFormatSharing( override def getFiles( table: Table, startingVersion: Long, - endingVersion: Option[Long] + endingVersion: Option[Long], + fileIdHash: Option[String] ): DeltaTableFiles = { assert( endingVersion.isDefined, @@ -214,6 +224,10 @@ private[spark] class TestClientForDeltaFormatSharing( ) val tableFullName = s"${table.share}.${table.schema}.${table.name}" TestClientForDeltaFormatSharing.requestedFormat.put(tableFullName, responseFormat) + TestClientForDeltaFormatSharing.fileIdHashHistory.synchronized { + TestClientForDeltaFormatSharing.fileIdHashHistory += + ((table.name, s"getFiles_streaming_${startingVersion}_${endingVersion.get}", fileIdHash)) + } val iterator = SparkEnv.get.blockManager .get[String](getBlockId(table.name, s"getFiles_${startingVersion}_${endingVersion.get}")) .map(_.data.asInstanceOf[Iterator[String]]) @@ -239,7 +253,8 @@ private[spark] class TestClientForDeltaFormatSharing( override def getCDFFiles( table: Table, cdfOptions: Map[String, String], - includeHistoricalMetadata: Boolean + includeHistoricalMetadata: Boolean, + fileIdHash: Option[String] ): DeltaTableFiles = { val suffix = cdfOptions .get(DeltaSharingOptions.CDF_START_VERSION) @@ -310,4 +325,16 @@ object TestClientForDeltaFormatSharing { val limits = scala.collection.mutable.Map[String, Long]() val requestedFormat = scala.collection.mutable.Map[String, String]() val jsonPredicateHints = scala.collection.mutable.Map[String, String]() + @volatile var lastCallerOrg: String = "" + + // Captures (tableName, queryType, fileIdHash) for each getFiles call. + val fileIdHashHistory = scala.collection.mutable.ArrayBuffer[(String, String, Option[String])]() + + def clearFileIdHashHistory(): Unit = fileIdHashHistory.synchronized { + fileIdHashHistory.clear() + } + + def getFileIdHashHistory: Seq[(String, String, Option[String])] = fileIdHashHistory.synchronized { + fileIdHashHistory.toSeq + } } diff --git a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceDeletionVectorsSuite.scala b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceDeletionVectorsSuite.scala new file mode 100644 index 00000000000..205297d57f6 --- /dev/null +++ b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceDeletionVectorsSuite.scala @@ -0,0 +1,86 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.test + +import org.apache.spark.sql.delta.DeltaSourceDeletionVectorsSuite +import org.apache.spark.sql.delta.sources.DeltaSQLConf + +/** + * Test suite that runs DeltaSourceDeletionVectorsSuite using the V2 connector + * (V2_ENABLE_MODE=STRICT). + */ +class DeltaV2SourceDeletionVectorsSuite + extends DeltaSourceDeletionVectorsSuite with V2ForceTest { + + override protected def useDsv2: Boolean = true + + /** + * Override executeDml to temporarily use V1 connector for DML operations. + * SparkTable (V2) is read-only and does not support writes, so DML must + * go through the V1 path. Only streaming reads use the V2 connector. + */ + override protected def executeDml(sqlText: String): Unit = { + withSQLConf(DeltaSQLConf.V2_ENABLE_MODE.key -> "NONE") { + sql(sqlText) + } + } + + private lazy val shouldPassTests = Set( + "allow to delete files before starting a streaming query", + "allow to delete files before staring a streaming query without checkpoint", + "multiple deletion vectors per file with initial snapshot", + "deleting files fails query if ignoreDeletes = false", + "deleting files when ignoreChanges = true doesn't fail the query", + "allow to delete files after staring a streaming query when ignoreDeletes is true", + "updating the source table causes failure when ignoreChanges = false - using DELETE", + "allow to update the source table when ignoreChanges = true - using DELETE", + "updating source table when ignoreDeletes = true fails the query - using DELETE", + "subsequent DML commands are processed correctly in a batch - DELETE->DELETE - List()", + "subsequent DML commands are processed correctly in a batch - DELETE->DELETE" + + " - List((ignoreDeletes,true))", + "subsequent DML commands are processed correctly in a batch - DELETE->DELETE" + + " - List((skipChangeCommits,true))", + "subsequent DML commands are processed correctly in a batch - DELETE->DELETE" + + " - List((ignoreChanges,true))", + "subsequent DML commands are processed correctly in a batch - INSERT->DELETE - List()", + "subsequent DML commands are processed correctly in a batch - INSERT->DELETE" + + " - List((ignoreDeletes,true))", + "subsequent DML commands are processed correctly in a batch - INSERT->DELETE" + + " - List((skipChangeCommits,true))", + "subsequent DML commands are processed correctly in a batch - INSERT->DELETE" + + " - List((ignoreChanges,true))", + "multiple deletion vectors per file - List((ignoreChanges,true))" + ) + + private lazy val shouldFailTests = Set( + // TODO(#5319): enable these tests after ignoreChanges/ignoreFileDeletion read options + // are supported by the V2 connector. + "allow to delete files after staring a streaming query when ignoreFileDeletion is true", + "multiple deletion vectors per file - List((ignoreFileDeletion,true))" + ) + + override protected def shouldFail(testName: String): Boolean = { + val inPassList = shouldPassTests.contains(testName) + val inFailList = shouldFailTests.contains(testName) + + assert(inPassList || inFailList, s"Test '$testName' not in shouldPassTests or shouldFailTests") + assert(!(inPassList && inFailList), + s"Test '$testName' in both shouldPassTests and shouldFailTests") + + inFailList + } +} diff --git a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala index 94fd9065a33..df87b30ba87 100644 --- a/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala +++ b/spark-unified/src/test/scala/org/apache/spark/sql/delta/test/DeltaV2SourceSuite.scala @@ -57,8 +57,22 @@ class DeltaV2SourceSuite extends DeltaSourceSuite with V2ForceTest { // === Schema Evolution === "add column: restarting with new DataFrame should recover", "add column: restarting with stale DataFrame should fail", + "relax nullability: restarting with new DataFrame should recover", + "type widening: restarting with new DataFrame should recover", "disallow to change schema after starting a streaming query", "allow to change schema before starting a streaming query", + "drop column: should fail with non-additive schema change error", + "drop column: should succeed with unsafe column mapping schema change flag enabled", + "rename column: should fail with non-additive schema change error", + "rename column: should throw schema change error with unsafe flag enabled", + "type widening: should fail with non-additive schema change error when enable schema tracking", + + // === Read options === + "excludeRegex works and doesn't mess up offsets across restarts - parquet version", + "streaming with ignoreDeletes = true skips delete-only commits", + "streaming with ignoreDeletes = true still fails on change commits", + "streaming with skipChangeCommits = true skips both delete and change commits", + "streaming with ignoreChanges = true allows both delete and change commits", // ========== startingVersion option tests ========== "startingVersion", @@ -93,7 +107,8 @@ class DeltaV2SourceSuite extends DeltaSourceSuite with V2ForceTest { // ========== Error handling tests ========== "streaming query should fail when table is deleted and recreated with new id", - "SC-46515: deltaSourceIgnoreDeleteError contains removeFile, version, tablePath", + "deltaSourceIgnoreDeleteError contains removeFile, version, tablePath", + "deltaSourceIgnoreChangesError contains removeFile, version, tablePath", "excludeRegex throws good error on bad regex pattern", // ========== Misc tests ========== @@ -109,16 +124,19 @@ class DeltaV2SourceSuite extends DeltaSourceSuite with V2ForceTest { "streaming delta source should not drop null columns", "streaming delta source should drop null columns without feature flag", - // === read options === - "skip change commits", - "excludeRegex works and doesn't mess up offsets across restarts - parquet version", - "SC-46515: deltaSourceIgnoreChangesError contains removeFile, version, tablePath", + // === Schema Evolution === + // TODO(#6232): enable the two tests after spark streaming engine supports leaf node projection + // for datasource v2 such that we can adopt the two schema changes without refreshing the + // dataframe + "relax nullability: restarting with stale DataFrame should recover", + "type widening: restarting with stale DataFrame should recover", // === Data Loss Detection === "fail on data loss - starting from missing files", "fail on data loss - gaps of files", "fail on data loss - starting from missing files with option off", "fail on data loss - gaps of files with option off", + "streaming processes 100 sequential single-value commits and contains all values 0 to 99", // === Misc === // TODO(#5900): fix exception mismatch diff --git a/spark/delta-suite-generator/src/main/scala/io/delta/suitegenerator/SuiteGeneratorConfig.scala b/spark/delta-suite-generator/src/main/scala/io/delta/suitegenerator/SuiteGeneratorConfig.scala index 568bef53330..cf18c009186 100644 --- a/spark/delta-suite-generator/src/main/scala/io/delta/suitegenerator/SuiteGeneratorConfig.scala +++ b/spark/delta-suite-generator/src/main/scala/io/delta/suitegenerator/SuiteGeneratorConfig.scala @@ -148,12 +148,6 @@ object SuiteGeneratorConfig { val DELETE_SCALA = DimensionMixin("DeleteScala", alias = Some("Scala")) val DELETE_SQL = DimensionMixin("DeleteSQL", alias = Some("SQL")) val DELETE_WITH_DVS = DimensionMixin("DeleteSQLWithDeletionVectors", alias = Some("DV")) - val STRUCT_EVOLUTION_PRESERVE_NULL_SOURCE = DimensionWithMultipleValues( - "StructEvolutionPreserveNullSource", - List("Disabled", "Enabled"), alias = Some("PreserveNullSource")) - val STRUCT_EVOLUTION_PRESERVE_NULL_SOURCE_UPDATE_STAR = DimensionWithMultipleValues( - "StructEvolutionPreserveNullSourceUpdateStar", - List("Disabled", "Enabled"), alias = Some("PreserveNullSourceUpdateStar")) } private object Tests { @@ -269,9 +263,7 @@ object SuiteGeneratorConfig { "MergeIntoStructEvolutionNullnessMultiClauseTests" :: Nil, List( List( - Dims.MERGE_SQL, Dims.NAME_BASED, Dims.COLUMN_MAPPING.asOptional, - Dims.STRUCT_EVOLUTION_PRESERVE_NULL_SOURCE, - Dims.STRUCT_EVOLUTION_PRESERVE_NULL_SOURCE_UPDATE_STAR + Dims.MERGE_SQL, Dims.NAME_BASED ) ) ) diff --git a/spark/src/main/resources/error/delta-error-classes.json b/spark/src/main/resources/error/delta-error-classes.json index d4411a75144..c9ca79fb9f6 100644 --- a/spark/src/main/resources/error/delta-error-classes.json +++ b/spark/src/main/resources/error/delta-error-classes.json @@ -2563,6 +2563,15 @@ ], "sqlState" : "42KD4" }, + "DELTA_STREAMING_INCOMPATIBLE_SCHEMA_CHANGE_V2" : { + "message" : [ + "Streaming read is not supported on tables with read-incompatible schema changes (e.g. rename or drop or datatype changes).", + "Non-additive schema change handling is not supported in Delta source v2 yet.", + "For further information and possible next steps to resolve this issue, please review the documentation at ", + "Read schema: . Incompatible data schema: ." + ], + "sqlState" : "42KD4" + }, "DELTA_STREAMING_INITIAL_SNAPSHOT_TOO_LARGE" : { "message" : [ "Initial snapshot for Delta streaming at table '' (version ) contains files, which exceeds the maximum allowed files.", @@ -3143,9 +3152,7 @@ }, "DELTA_UNSUPPORTED_PARTITION_COLUMN_CHANGE" : { "message" : [ - "Cannot change partition columns during operation.", - "- Old partition columns: .", - "- New partition columns: ." + "Cannot change partition columns during operation (old: , new: )." ], "sqlState" : "42P10" }, @@ -3253,6 +3260,12 @@ ], "sqlState" : "42846" }, + "DELTA_USER_DEFINED_TYPE_COLUMN_CONTAINS_NULL_TYPE" : { + "message" : [ + "Found NullType in column which is of user-defined type. Delta doesn't support writing NullType in user-defined types." + ], + "sqlState" : "22005" + }, "DELTA_VACUUM_RETENTION_PERIOD_NEGATIVE" : { "message" : [ "Retention period for Vacuum can't be less than 0 hours." diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/Checksum.scala b/spark/src/main/scala/org/apache/spark/sql/delta/Checksum.scala index de7d32d9a74..33a27cca4d6 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/Checksum.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/Checksum.scala @@ -86,6 +86,54 @@ case class VersionChecksum( @JsonAlias(Array("fileSizeHistogram")) histogramOpt: Option[FileSizeHistogram], deletedRecordCountsHistogramOpt: Option[DeletedRecordCountsHistogram], + allFiles: Option[Seq[AddFile]]) { + + /** + * Converts to the protocol-compliant representation that serializes the histogram field as + * `fileSizeHistogram` (the Delta spec field name) instead of `histogramOpt`. + */ + def toProtocolCompliant: VersionChecksumProtocolCompliant = VersionChecksumProtocolCompliant( + txnId = txnId, + tableSizeBytes = tableSizeBytes, + numFiles = numFiles, + numDeletedRecordsOpt = numDeletedRecordsOpt, + numDeletionVectorsOpt = numDeletionVectorsOpt, + numMetadata = numMetadata, + numProtocol = numProtocol, + inCommitTimestampOpt = inCommitTimestampOpt, + setTransactions = setTransactions, + domainMetadata = domainMetadata, + metadata = metadata, + protocol = protocol, + fileSizeHistogram = histogramOpt, + deletedRecordCountsHistogramOpt = deletedRecordCountsHistogramOpt, + allFiles = allFiles + ) +} + +/** + * Protocol-compliant version of [[VersionChecksum]] that serializes the file size histogram + * using the Delta spec field name `fileSizeHistogram` instead of the legacy `histogramOpt`. + * Used only for CRC file writes when the protocol-compliant flag is enabled. + */ +case class VersionChecksumProtocolCompliant( + txnId: Option[String], + tableSizeBytes: Long, + numFiles: Long, + @JsonDeserialize(contentAs = classOf[Long]) + numDeletedRecordsOpt: Option[Long], + @JsonDeserialize(contentAs = classOf[Long]) + numDeletionVectorsOpt: Option[Long], + numMetadata: Long, + numProtocol: Long, + @JsonDeserialize(contentAs = classOf[Long]) + inCommitTimestampOpt: Option[Long], + setTransactions: Option[Seq[SetTransaction]], + domainMetadata: Option[Seq[DomainMetadata]], + metadata: Metadata, + protocol: Protocol, + fileSizeHistogram: Option[FileSizeHistogram], + deletedRecordCountsHistogramOpt: Option[DeletedRecordCountsHistogram], allFiles: Option[Seq[AddFile]]) /** @@ -105,15 +153,23 @@ trait RecordChecksum extends DeltaLogging { return } - val version = snapshot.version val checksumWithoutTxnId = getChecksum(snapshot) val checksum = checksumWithoutTxnId.copy(txnId = Some(txnId)) + writeChecksumFile(snapshot.version, checksum) + } + + protected def writeChecksumFile(version: Long, checksum: VersionChecksum): Unit = { val eventData = mutable.Map[String, Any]("operationSucceeded" -> false) eventData("numAddFileActions") = checksum.allFiles.map(_.size).getOrElse(-1) eventData("numSetTransactionActions") = checksum.setTransactions.map(_.size).getOrElse(-1) val startTimeMs = System.currentTimeMillis() try { - val toWrite = JsonUtils.toJson(checksum) + "\n" + val toWrite = (if (spark.conf.get( + DeltaSQLConf.DELTA_CHECKSUM_HISTOGRAM_FIELD_FOLLOWS_PROTOCOL)) { + JsonUtils.toJson(checksum.toProtocolCompliant) + } else { + JsonUtils.toJson(checksum) + }) + "\n" eventData("jsonSerializationTimeTakenMs") = System.currentTimeMillis() - startTimeMs eventData("checksumLength") = toWrite.length val stream = writer.createAtomic( diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaAnalysis.scala b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaAnalysis.scala index 19bfeae4345..96b7d4aaa9a 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaAnalysis.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaAnalysis.scala @@ -101,7 +101,12 @@ class DeltaAnalysis(session: SparkSession) // since we only want to up-cast for SQL insert into by name case a @ AppendDelta(r, d) if a.isByName && a.origin.sqlText.nonEmpty && needsSchemaAdjustmentByName(a.query, r.output, d, a.writeOptions) => - val projection = resolveQueryColumnsByName(a.query, r.output, d, a.writeOptions) + val projection = resolveQueryColumnsByName( + query = a.query, + targetAttrs = r.output, + deltaTable = d, + writeOptions = a.writeOptions, + allowSchemaEvolution = true) if (projection != a.query) { a.copy(query = projection) } else { @@ -246,7 +251,10 @@ class DeltaAnalysis(session: SparkSession) protocol } val newDeltaCatalog = new DeltaCatalogV1() - val existingTableOpt = newDeltaCatalog.getExistingTableIfExists(catalogTableTarget.identifier) + val existingTableOpt = newDeltaCatalog.getExistingTableIfExists( + catalogTableTarget.identifier, + identOpt = None, + operation = TableCreationModes.Create) val newTable = newDeltaCatalog .verifyTableAndSolidify( catalogTableTarget, @@ -282,7 +290,12 @@ class DeltaAnalysis(session: SparkSession) // since we only want to up-cast for SQL insert into by name case o @ OverwriteDelta(r, d) if o.isByName && o.origin.sqlText.nonEmpty && needsSchemaAdjustmentByName(o.query, r.output, d, o.writeOptions) => - val projection = resolveQueryColumnsByName(o.query, r.output, d, o.writeOptions) + val projection = resolveQueryColumnsByName( + query = o.query, + targetAttrs = r.output, + deltaTable = d, + writeOptions = o.writeOptions, + allowSchemaEvolution = true) if (projection != o.query) { val aliases = AttributeMap(o.query.output.zip(projection.output).collect { case (l: AttributeReference, r: AttributeReference) if !l.sameRef(r) => (l, r) @@ -308,7 +321,12 @@ class DeltaAnalysis(session: SparkSession) // INSERT OVERWRITE by name // OverwriteDelta.byName is also used for DataFrame append so we check for the SQL origin // text since we only want to up-cast for SQL insert into by name - resolveQueryColumnsByName(o.query, r.output, d, o.writeOptions) + resolveQueryColumnsByName( + query = o.query, + targetAttrs = r.output, + deltaTable = d, + writeOptions = o.writeOptions, + allowSchemaEvolution = true) } else { o.query } @@ -973,12 +991,19 @@ class DeltaAnalysis(session: SparkSession) query: LogicalPlan, targetAttrs: Seq[Attribute], deltaTable: DeltaTableV2, - writeOptions: Map[String, String]): LogicalPlan = { - insertIntoByNameMissingColumn(query, targetAttrs, deltaTable) + writeOptions: Map[String, String], + allowSchemaEvolution: Boolean = false): LogicalPlan = { + // Schema evolution is only effective when mergeSchema is enabled in write options AND + // the feature is enabled via SQL conf. + val effectiveSchemaEvolution = allowSchemaEvolution && + new DeltaOptions(deltaTable.options ++ writeOptions, conf).canMergeSchema && + session.conf.get(DeltaSQLConf.DELTA_INSERT_BY_NAME_SCHEMA_EVOLUTION_ENABLED) + + insertIntoByNameMissingColumn(query, targetAttrs, deltaTable, effectiveSchemaEvolution) // This is called before resolveOutputColumns in postHocResolutionRules, so we need to duplicate // the schema validation here. - if (query.output.length > targetAttrs.length) { + if (!effectiveSchemaEvolution && query.output.length > targetAttrs.length) { throw QueryCompilationErrors.cannotWriteTooManyColumnsToTableError( tableName = deltaTable.name(), expected = targetAttrs.map(_.name), @@ -988,7 +1013,12 @@ class DeltaAnalysis(session: SparkSession) val project = query.output.map { attr => val targetAttr = targetAttrs.find(t => session.sessionState.conf.resolver(t.name, attr.name)) .getOrElse { - throw DeltaErrors.missingColumn(attr, targetAttrs) + if (effectiveSchemaEvolution) { + attr + } else { + // Extra columns in the source are not allowed when schema evolution is disabled. + throw DeltaErrors.missingColumn(attr, targetAttrs) + } } addCastToColumn(attr, targetAttr, deltaTable.name(), typeWideningMode = getTypeWideningMode(deltaTable, writeOptions) @@ -1081,10 +1111,20 @@ class DeltaAnalysis(session: SparkSession) private def insertIntoByNameMissingColumn( query: LogicalPlan, targetAttrs: Seq[Attribute], - deltaTable: DeltaTableV2): Unit = { - if (query.output.length < targetAttrs.length) { - // Some columns are not specified. We don't allow schema evolution in INSERT INTO BY NAME, so - // we need to ensure the missing columns must be generated columns. + deltaTable: DeltaTableV2, + allowSchemaEvolution: Boolean = false): Unit = { + // When allowing the source schema to contain extra columns, it can still + // be missing required columns from the target schema. + // + // The total column count alone is not sufficient for validation. + // A source may have more columns overall but still omit specific columns + // that are required by the target schema. + // + // Example: + // Target: [a, b] + // Source: [a, x, y] + // Since the source has 3 columns vs target's 2, but is missing column b, this should be caught. + if (allowSchemaEvolution || query.output.length < targetAttrs.length) { val userSpecifiedNames = if (session.sessionState.conf.caseSensitiveAnalysis) { query.output.map(a => (a.name, a)).toMap } else { @@ -1546,4 +1586,3 @@ case class DeltaDynamicPartitionOverwriteCommand( ).run(sparkSession) } } - diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaColumnMapping.scala b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaColumnMapping.scala index 672731e77d3..e4d373b524b 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaColumnMapping.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaColumnMapping.scala @@ -657,17 +657,19 @@ trait DeltaColumnMappingBase extends DeltaLogging { * We detect DROP COLUMNS by checking if any physical name in `currentSchema` is missing in * `newSchema`. */ - def isDropColumnOperation(newMetadata: Metadata, currentMetadata: Metadata): Boolean = { + def isDropColumnOperation( + newSchema: StructType, + currentSchema: StructType, + isBothColumnMappingEnabled: Boolean): Boolean = { // We will need to compare the new schema's physical columns to the current schema's physical // columns. So, they both must have column mapping enabled. - if (newMetadata.columnMappingMode == NoMapping || - currentMetadata.columnMappingMode == NoMapping) { + if (!isBothColumnMappingEnabled) { return false } - val newPhysicalToLogicalMap = getPhysicalNameFieldMap(newMetadata.schema) - val currentPhysicalToLogicalMap = getPhysicalNameFieldMap(currentMetadata.schema) + val newPhysicalToLogicalMap = getPhysicalNameFieldMap(newSchema) + val currentPhysicalToLogicalMap = getPhysicalNameFieldMap(currentSchema) // are any of the current physical names missing in the new schema? currentPhysicalToLogicalMap @@ -704,17 +706,19 @@ trait DeltaColumnMappingBase extends DeltaLogging { * We detect RENAME COLUMNS by checking if any two columns with the same physical name have * different logical names */ - def isRenameColumnOperation(newMetadata: Metadata, currentMetadata: Metadata): Boolean = { + def isRenameColumnOperation( + newSchema: StructType, + currentSchema: StructType, + isBothColumnMappingEnabled: Boolean): Boolean = { // We will need to compare the new schema's physical columns to the current schema's physical // columns. So, they both must have column mapping enabled. - if (newMetadata.columnMappingMode == NoMapping || - currentMetadata.columnMappingMode == NoMapping) { + if (!isBothColumnMappingEnabled) { return false } - val newPhysicalToLogicalMap = getPhysicalNameFieldMap(newMetadata.schema) - val currentPhysicalToLogicalMap = getPhysicalNameFieldMap(currentMetadata.schema) + val newPhysicalToLogicalMap = getPhysicalNameFieldMap(newSchema) + val currentPhysicalToLogicalMap = getPhysicalNameFieldMap(currentSchema) // do any two columns with the same physical name have different logical names? currentPhysicalToLogicalMap @@ -723,6 +727,21 @@ trait DeltaColumnMappingBase extends DeltaLogging { } } + /** + * Returns true if there is a column mapping schema change (drop/rename) or an incompatible + * partition column change between the new and current schemas. + */ + def hasColMappingOrPartitionSchemaChange( + newSchema: StructType, + currentSchema: StructType, + newPartitionColumns: Seq[String], + oldPartitionColumns: Seq[String], + isBothColumnMappingEnabled: Boolean): Boolean = { + isDropColumnOperation(newSchema, currentSchema, isBothColumnMappingEnabled) || + isRenameColumnOperation(newSchema, currentSchema, isBothColumnMappingEnabled) || + !SchemaUtils.isPartitionCompatible(newPartitionColumns, oldPartitionColumns) + } + /** * Collects the column rename operations between the new schema and the current schema. * @param newSchema The new schema after a potential rename. @@ -758,22 +777,24 @@ trait DeltaColumnMappingBase extends DeltaLogging { */ def hasNoColumnMappingSchemaChanges(newMetadata: Metadata, oldMetadata: Metadata, allowUnsafeReadOnPartitionChanges: Boolean = false): Boolean = { - // Helper function to check no column mapping schema change and no repartition - def hasNoColMappingAndRepartitionSchemaChange( - newMetadata: Metadata, oldMetadata: Metadata): Boolean = { - isRenameColumnOperation(newMetadata, oldMetadata) || - isDropColumnOperation(newMetadata, oldMetadata) || - !SchemaUtils.isPartitionCompatible( - // if allow unsafe row read for partition change, ignore the check - if (allowUnsafeReadOnPartitionChanges) Seq.empty else newMetadata.partitionColumns, - if (allowUnsafeReadOnPartitionChanges) Seq.empty else oldMetadata.partitionColumns) + def hasColMappingOrPartitionSchemaChangeByMetadata(newMetadata: Metadata, + oldMetadata: Metadata): Boolean = { + val isBothColumnMappingEnabled = + newMetadata.columnMappingMode != NoMapping && oldMetadata.columnMappingMode != NoMapping + hasColMappingOrPartitionSchemaChange( + newMetadata.schema, + oldMetadata.schema, + // if allow unsafe row read for partition change, ignore the check + if (allowUnsafeReadOnPartitionChanges) Seq.empty else newMetadata.partitionColumns, + if (allowUnsafeReadOnPartitionChanges) Seq.empty else oldMetadata.partitionColumns, + isBothColumnMappingEnabled) } val (oldMode, newMode) = (oldMetadata.columnMappingMode, newMetadata.columnMappingMode) if (oldMode != NoMapping && newMode != NoMapping) { require(oldMode == newMode, "changing mode is not supported") // Both changes are post column mapping enabled - !hasNoColMappingAndRepartitionSchemaChange(newMetadata, oldMetadata) + !hasColMappingOrPartitionSchemaChangeByMetadata(newMetadata, oldMetadata) } else if (oldMode == NoMapping && newMode != NoMapping) { // The old metadata does not have column mapping while the new metadata does, in this case // we assume an upgrade has happened in between. @@ -790,7 +811,7 @@ trait DeltaColumnMappingBase extends DeltaLogging { Map(DeltaConfigs.COLUMN_MAPPING_MODE.key -> newMetadata.columnMappingMode.name) ) // use the same check - !hasNoColMappingAndRepartitionSchemaChange(newMetadata, upgradedMetadata) + !hasColMappingOrPartitionSchemaChangeByMetadata(newMetadata, upgradedMetadata) } else { // Prohibit reading across a downgrade. val isDowngrade = oldMode != NoMapping && newMode == NoMapping diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaConfig.scala b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaConfig.scala index 5c9830dbf3c..2d20f4d1cd8 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaConfig.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaConfig.scala @@ -850,6 +850,18 @@ trait DeltaConfigsBase extends DeltaLogging { validationFunction = _ => true, helpMessage = "needs to be a boolean." ) + /** + * Guard property automatically set when a new IcebergCompat table is created + * Atomic UniForm Iceberg conversion requires this property to be present + */ + val ICEBERG_ATOMIC_CONVERSION_SUPPORTED = buildConfig[Boolean]( + "universalFormat.iceberg.atomicConversion.supported", + "false", + _.toBoolean, + _ => true, + "needs to be a boolean.", + userConfigurable = true + ) val CAST_ICEBERG_TIME_TYPE = buildConfig[Boolean]( key = "castIcebergTimeType", diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaErrors.scala b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaErrors.scala index ca622d0b40d..aa7a96cef17 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaErrors.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaErrors.scala @@ -3221,7 +3221,8 @@ trait DeltaErrorsBase spark: SparkSession, readSchema: StructType, incompatibleSchema: StructType, - detectedDuringStreaming: Boolean): Throwable = { + detectedDuringStreaming: Boolean, + isV2DataSource: Boolean = false): Throwable = { val docLink = "/versioning.html#column-mapping" val enableNonAdditiveSchemaEvolution = spark.sessionState.conf.getConf( DeltaSQLConf.DELTA_STREAMING_ENABLE_SCHEMA_TRACKING) @@ -3231,7 +3232,8 @@ trait DeltaErrorsBase generateDocsLinkOption(spark, docLink).getOrElse("-"), enableNonAdditiveSchemaEvolution, additionalProperties = Map( - "detectedDuringStreaming" -> detectedDuringStreaming.toString + "detectedDuringStreaming" -> detectedDuringStreaming.toString, + "isV2DataSource" -> isV2DataSource.toString )) } @@ -3540,8 +3542,8 @@ trait DeltaErrorsBase messageParameters = Array( UniversalFormat.ICEBERG_FORMAT, "Requires IcebergCompat to be explicitly enabled in order for Universal Format (Iceberg) " + - "to be enabled on an existing table. To enable IcebergCompatV2, set the table property " + - "'delta.enableIcebergCompatV2' = 'true'." + "to be enabled on an existing table. Supported versions are IcebergCompatV1 and " + + "IcebergCompatV2." ) ) } @@ -4358,7 +4360,9 @@ class DeltaStreamingNonAdditiveSchemaIncompatibleException( val enableNonAdditiveSchemaEvolution: Boolean = false, val additionalProperties: Map[String, String] = Map.empty) extends DeltaUnsupportedOperationException( - errorClass = if (enableNonAdditiveSchemaEvolution) { + errorClass = if (additionalProperties.getOrElse("isV2DataSource", "false") == "true") { + "DELTA_STREAMING_INCOMPATIBLE_SCHEMA_CHANGE_V2" + } else if (enableNonAdditiveSchemaEvolution) { "DELTA_STREAMING_INCOMPATIBLE_SCHEMA_CHANGE_USE_SCHEMA_LOG" } else { "DELTA_STREAMING_INCOMPATIBLE_SCHEMA_CHANGE" diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaLog.scala b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaLog.scala index d6b6d19cd69..2f10c4bf371 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaLog.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaLog.scala @@ -28,6 +28,7 @@ import scala.util.Try import scala.util.control.NonFatal import com.databricks.spark.util.TagDefinitions._ +import org.apache.spark.sql.delta.v2.interop.DeltaV2TableManager import org.apache.spark.sql.delta.DataFrameUtils import org.apache.spark.sql.delta.ClassicColumnConversions._ import org.apache.spark.sql.delta.actions._ @@ -88,7 +89,8 @@ class DeltaLog private( with SnapshotManagement with DeltaFileFormat with ProvidesUniFormConverters - with ReadChecksum { + with ReadChecksum + with DeltaV2TableManager { import org.apache.spark.sql.delta.files.TahoeFileIndex import org.apache.spark.sql.delta.util.FileNames._ diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaOptions.scala b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaOptions.scala index f42afd5c9ed..de2792546e2 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaOptions.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaOptions.scala @@ -53,6 +53,11 @@ trait DeltaWriteOptions val replaceWhere: Option[String] = options.get(REPLACE_WHERE_OPTION) val userMetadata: Option[String] = options.get(USER_METADATA_OPTION) + val useNullIntolerantEqualityWithDPO: Option[Boolean] = + options + .get(USE_NULL_INTOLERANT_EQUALITY_WITH_DPO) + .map(toBoolean(_, USE_NULL_INTOLERANT_EQUALITY_WITH_DPO)) + /** * Whether to add an adaptive shuffle before writing out the files to break skew, and coalesce * data into chunkier files. @@ -259,6 +264,12 @@ object DeltaOptions extends DeltaLogging { /** An option to specify user-defined metadata in commitInfo */ val USER_METADATA_OPTION = "userMetadata" + /** + * An option to ignore overwriting partitions that contain NULL in + * Dynamic Partition Overwrite, used by INSERT INTO ... REPLACE USING (...). + */ + val USE_NULL_INTOLERANT_EQUALITY_WITH_DPO = "useNullIntolerantEqualityWithDPO" + val PARTITION_OVERWRITE_MODE_OPTION = "partitionOverwriteMode" val PARTITION_OVERWRITE_MODE_DYNAMIC = "DYNAMIC" val PARTITION_OVERWRITE_MODE_STATIC = "STATIC" @@ -325,6 +336,7 @@ object DeltaOptions extends DeltaLogging { EXCLUDE_REGEX_OPTION, OVERWRITE_SCHEMA_OPTION, USER_METADATA_OPTION, + USE_NULL_INTOLERANT_EQUALITY_WITH_DPO, PARTITION_OVERWRITE_MODE_OPTION, MAX_FILES_PER_TRIGGER_OPTION, IGNORE_FILE_DELETION_OPTION, diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaTableValueFunctions.scala b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaTableValueFunctions.scala index c8e4ea9fd9b..f25479a4023 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/DeltaTableValueFunctions.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/DeltaTableValueFunctions.scala @@ -26,10 +26,11 @@ import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.sources.DeltaDataSource +import org.apache.spark.SparkException import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.FunctionIdentifier import org.apache.spark.sql.catalyst.analysis.{FunctionRegistryBase, NamedRelation, TableFunctionRegistry, UnresolvedLeafNode, UnresolvedRelation} -import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, ExpressionInfo, Literal, StringLiteral, Unevaluable} +import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, ExpressionInfo, Literal, StringLiteral} import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan, UnaryNode} import org.apache.spark.sql.connector.catalog.V1Table import org.apache.spark.sql.execution.datasources.LogicalRelation @@ -114,12 +115,11 @@ trait CDCStatementBase extends DeltaTableValueFunction { val fakePlan = util.AnalysisHelper.FakeLogicalPlan(Seq(value), Nil) val timestampExpression = org.apache.spark.sql.catalyst.optimizer.ComputeCurrentTime(fakePlan).expressions.head - if (timestampExpression.isInstanceOf[Unevaluable]) { - throw DeltaErrors.cdcNonConstantArgument(fnName, keyPrefix, position, value) - } timestampExpression.eval().toString } catch { case _: NullPointerException => throw DeltaErrors.nullRangeBoundaryInCDCRead() + case e: SparkException if e.getErrorClass == "INTERNAL_ERROR" => + throw DeltaErrors.cdcNonConstantArgument(fnName, keyPrefix, position, value) } value.dataType match { // We dont need to explicitly handle ShortType as it is parsed as IntegerType. diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/IcebergCompat.scala b/spark/src/main/scala/org/apache/spark/sql/delta/IcebergCompat.scala index f71222a68f1..f99d0c26c22 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/IcebergCompat.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/IcebergCompat.scala @@ -46,7 +46,7 @@ object IcebergCompatV1 extends IcebergCompatBase( icebergFormatVersion = 2, config = DeltaConfigs.ICEBERG_COMPAT_V1_ENABLED, tableFeature = IcebergCompatV1TableFeature, - requiredTableProperties = Seq(RequireColumnMapping), + relatedTableProperties = Seq(RequireColumnMapping, OptionalAtomicEligible), incompatibleTableFeatures = Set(DeletionVectorsTableFeature), checks = Seq( CheckOnlySingleVersionEnabled, @@ -63,7 +63,7 @@ object IcebergCompatV2 extends IcebergCompatBase( icebergFormatVersion = 2, config = DeltaConfigs.ICEBERG_COMPAT_V2_ENABLED, tableFeature = IcebergCompatV2TableFeature, - requiredTableProperties = Seq(RequireColumnMapping), + relatedTableProperties = Seq(RequireColumnMapping, OptionalAtomicEligible), incompatibleTableFeatures = Set(DeletionVectorsTableFeature), checks = Seq( CheckOnlySingleVersionEnabled, @@ -95,7 +95,7 @@ case class IcebergCompatBase( icebergFormatVersion: Int, config: DeltaConfig[Option[Boolean]], tableFeature: TableFeature, - requiredTableProperties: Seq[RequiredDeltaTableProperty[_<:Any]], + relatedTableProperties: Seq[DeltaTableProperty], incompatibleTableFeatures: Set[TableFeature] = Set.empty, checks: Seq[IcebergCompatCheck]) extends DeltaLogging { def isEnabled(metadata: Metadata): Boolean = config.fromMetaData(metadata).getOrElse(false) @@ -156,8 +156,10 @@ case class IcebergCompatBase( } } - // Check we have all required delta table properties - requiredTableProperties.foreach { + // Go through all relatedProperties: + // 1. Check we have all required delta table properties + // 2. Enable related non-required delta properties if needed + relatedTableProperties.foreach { case RequiredDeltaTableProperty( deltaConfig, validator, autoSetValue, autoEnableOnExistingTable) => val newestValue = deltaConfig.fromMetaData(newestMetadata) @@ -179,6 +181,24 @@ case class IcebergCompatBase( deltaConfig.key, newestValue.toString, autoSetValue) } } + case OptionalDeltaTableProperty( + deltaConfig, + autoEnableOnExistingTables, + autoSetValue, + shouldExistingValuesBeingPreserved + ) => + val newestValueExplicitlySet = newestMetadata.configuration.contains(deltaConfig.key) + if (!newestValueExplicitlySet && + (autoEnableOnExistingTables || isCreatingOrReorgTable) + ) { + val oldValueExplicitlySet = prevMetadata.configuration.contains(deltaConfig.key) + val setValue = if (shouldExistingValuesBeingPreserved && oldValueExplicitlySet) { + prevMetadata.configuration(deltaConfig.key) + } else { + autoSetValue + } + tblPropertyUpdates += deltaConfig.key -> setValue + } } // Update Protocol and Metadata if necessary @@ -196,7 +216,7 @@ case class IcebergCompatBase( val newConfiguration = newestMetadata.configuration ++ tblPropertyUpdates.toMap var tmpNewMetadata = newestMetadata.copy(configuration = newConfiguration) - requiredTableProperties.foreach { tp => + relatedTableProperties.foreach { tp => tmpNewMetadata = tp.postProcess(prevMetadata, tmpNewMetadata, isCreatingOrReorgTable) } @@ -302,6 +322,16 @@ object IcebergCompat extends IcebergCompatVersionBase( ) with DeltaLogging +trait DeltaTableProperty { + /** + * A callback after all properties are added to the new metadata. + * @return Updated metadata. None if no change + */ + def postProcess( + prevMetadata: Metadata, + newMetadata: Metadata, + isCreatingNewTable: Boolean) : Metadata = newMetadata +} /** * Wrapper class for table property validation @@ -316,16 +346,14 @@ case class RequiredDeltaTableProperty[T]( deltaConfig: DeltaConfig[T], validator: T => Boolean, autoSetValue: String, - autoEnableOnExistingTable: Boolean = false) { - /** - * A callback after all required properties are added to the new metadata. - * @return Updated metadata. None if no change - */ - def postProcess( - prevMetadata: Metadata, - newMetadata: Metadata, - isCreatingNewTable: Boolean) : Metadata = newMetadata -} + autoEnableOnExistingTable: Boolean = false) extends DeltaTableProperty + +case class OptionalDeltaTableProperty[T]( + deltaConfig: DeltaConfig[T], + autoEnableOnExistingTables: Boolean, + autoSetValue: String, + shouldExistingValuesBeingPreserved: Boolean) extends DeltaTableProperty + class RequireColumnMapping(allowedModes: Seq[DeltaColumnMappingMode]) extends RequiredDeltaTableProperty( @@ -355,6 +383,13 @@ class RequireColumnMapping(allowedModes: Seq[DeltaColumnMappingMode]) object RequireColumnMapping extends RequireColumnMapping(Seq(NameMapping, IdMapping)) +object OptionalAtomicEligible extends OptionalDeltaTableProperty( + deltaConfig = DeltaConfigs.ICEBERG_ATOMIC_CONVERSION_SUPPORTED, + autoEnableOnExistingTables = false, + autoSetValue = "true", + shouldExistingValuesBeingPreserved = true +) + case class IcebergCompatContext( spark: SparkSession, diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala b/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala index b83d921d4db..9cfcbecc39d 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/OptimisticTransaction.scala @@ -35,9 +35,11 @@ import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.DeletionVectorUtils import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.constraints.{Constraints, Invariants} -import org.apache.spark.sql.delta.coordinatedcommits.{CatalogOwnedTableUtils, CoordinatedCommitsUtils, TableCommitCoordinatorClient} +import org.apache.spark.sql.delta.coordinatedcommits.{CatalogOwnedTableUtils, CoordinatedCommitsUtils, TableCommitCoordinatorClient, UCCommitCoordinatorBuilder} import org.apache.spark.sql.delta.files._ import org.apache.spark.sql.delta.hooks.{CheckpointHook, ChecksumHook, GenerateSymlinkManifest, HudiConverterHook, IcebergConverterHook, PostCommitHook, UpdateCatalogFactory} +import org.apache.spark.sql.delta.hooks.metrics.UpdateMetricsHook +import org.apache.spark.sql.delta.util.CatalogTableUtils import org.apache.spark.sql.delta.implicits.addFileEncoder import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging @@ -57,11 +59,12 @@ import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.spark.SparkException import org.apache.spark.internal.{MDC, MessageWithContext} import org.apache.spark.sql.{AnalysisException, Column, DataFrame, SaveMode, SparkSession} -import org.apache.spark.sql.catalyst.catalog.CatalogTable +import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogTableType} import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.plans.logical.UnsetTableProperties import org.apache.spark.sql.catalyst.types.DataTypeUtils.toAttributes import org.apache.spark.sql.catalyst.util.{CharVarcharUtils, ResolveDefaultColumns} +import org.apache.spark.sql.delta.clustering.ClusteringMetadataDomain import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{StructField, StructType} import org.apache.spark.util.{Clock, Utils} @@ -448,6 +451,9 @@ trait OptimisticTransactionImpl extends TransactionHelper registerPostCommitHook(ChecksumHook) catalogTable.foreach { ct => registerPostCommitHook(UpdateCatalogFactory.getUpdateCatalogHook(ct, spark)) + if (CatalogTableUtils.isUnityCatalogManagedTable(ct)) { + registerPostCommitHook(UpdateMetricsHook(Some(ct))) + } } // The CheckpointHook will only checkpoint if necessary, so always register it to run. registerPostCommitHook(CheckpointHook) @@ -574,6 +580,80 @@ trait OptimisticTransactionImpl extends TransactionHelper assert(newMetadata.isEmpty, "Cannot change the metadata more than once in a transaction.") updateMetadataInternal(proposedNewMetadata, ignoreDefaultProperties) + // Temporary: block metadata changes on UC-managed CatalogOwned tables until Delta supports + // propagating metadata updates to UC. UC is identified by catalog implementation class (handles + // "spark_catalog" registration). New table creation is naturally excluded because + // isCatalogOwned is false until the first commit. REPLACE TABLE is currently also blocked + // here and will need to be explicitly allowed once UC supports metadata propagation. + // Intentionally conservative: configuration is compared as a whole map, which also + // catches Delta-internal additions (e.g. table-feature flags). This is acceptable for + // a temporary kill switch - once Delta supports propagating metadata updates to UC, + // this check will be removed entirely. + if (!isCreatingNewTable) { + throwIfUCManagedMetadataChanged(snapshot.metadata, context = "updateMetadata") + } + } + + /** + * Returns true if the proposed metadata differs from the existing metadata for a UC-managed + * table. + */ + private def hasUCManagedMetadataChange( + existingMetadata: Metadata, + proposedMetadata: Metadata): Boolean = { + proposedMetadata.schemaString != existingMetadata.schemaString || + proposedMetadata.partitionColumns != existingMetadata.partitionColumns || + proposedMetadata.description != existingMetadata.description || + proposedMetadata.configuration != existingMetadata.configuration + } + + private def throwIfUCManagedMetadataChanged( + existingMetadata: Metadata, + context: String): Unit = { + val proposedMetadata = newMetadata.getOrElse(existingMetadata) + if (isUCManagedTable && hasUCManagedMetadataChange(existingMetadata, proposedMetadata)) { + logWarning(log"Blocking UC-managed metadata update during " + + log"${MDC(DeltaLogKeys.OPERATION, context)} because metadata changed: " + + log"${MDC(DeltaLogKeys.METADATA_OLD, existingMetadata)} => " + + log"${MDC(DeltaLogKeys.METADATA_NEW, proposedMetadata)}") + throw DeltaErrors.operationNotSupportedException( + "Metadata changes on Unity Catalog managed tables") + } + } + + /** + * True if this transaction targets a UC-managed CatalogOwned table. + * + * Computed once as a lazy val because catalogTable and SparkSession are immutable for + * the lifetime of a transaction. Visibility is protected[delta] (not private) to allow + * test subclasses to override without requiring UCSingleCatalog. + */ + protected[delta] lazy val isUCManagedTable: Boolean = { + snapshot.isCatalogOwned && + catalogTable.exists { ct => + ct.tableType == CatalogTableType.MANAGED && + CatalogOwnedTableUtils.getCatalogName(spark, ct.identifier) + .contains(UCCommitCoordinatorBuilder.COORDINATOR_NAME) + } + } + + /** + * Returns true if committing [[dm]] would change the clustering columns on a UC-managed + * CatalogOwned table and should therefore be blocked. + * + * Both a missing entry and a removed=true tombstone mean "no clustering", so the effective + * configuration is normalised to Option[String] before comparison. + */ + private def isClusteringChangedOnUCManagedTable(dm: DomainMetadata): Boolean = { + if (dm.domain != ClusteringMetadataDomain.domainName) return false + if (!isUCManagedTable) return false + val existingConfig = + snapshot.domainMetadata + .find(_.domain == ClusteringMetadataDomain.domainName) + .filterNot(_.removed) + .map(_.configuration) + val incomingConfig = if (dm.removed) None else Some(dm.configuration) + incomingConfig != existingConfig } /** @@ -874,6 +954,8 @@ trait OptimisticTransactionImpl extends TransactionHelper CoordinatedCommitsUtils.getExplicitCCConfigurations(snapshot.metadata.configuration) val existingICTConfs = CoordinatedCommitsUtils.getExplicitICTConfigurations(snapshot.metadata.configuration) + val existingQoLConfs = + CoordinatedCommitsUtils.getExplicitQoLConfigurations(snapshot.metadata.configuration) val oldMappingMode = snapshot.metadata.columnMappingMode val newMappingMode = metadata.columnMappingMode val shouldReuseColumnMetadataForReplaceTable = @@ -889,8 +971,11 @@ trait OptimisticTransactionImpl extends TransactionHelper // to remove them and retain the Coordinated Commits configurations from the existing table. val newConfsWithoutCC = newMetadata.get.configuration -- CoordinatedCommitsUtils.TABLE_PROPERTY_KEYS - var newConfs: Map[String, String] = newConfsWithoutCC ++ existingCCConfs ++ - existingUCTableIdConf + val existingQoLConfsToRetain = existingQoLConfs.filterNot { case (key, _) => + newConfsWithoutCC.contains(key) + } + var newConfs: Map[String, String] = + newConfsWithoutCC ++ existingCCConfs ++ existingQoLConfsToRetain ++ existingUCTableIdConf // We also need to retain the existing ICT dependency configurations, but only when the // existing table does have Coordinated Commits configurations or Catalog-Owned enabled. // Otherwise, we treat the ICT configurations the same as any other configurations, @@ -902,6 +987,9 @@ trait OptimisticTransactionImpl extends TransactionHelper newConfs = newConfsWithoutICT ++ existingICTConfs } newMetadata = Some(newMetadata.get.copy(configuration = newConfs)) + throwIfUCManagedMetadataChanged( + snapshot.metadata, + context = "updateMetadataForNewTableInReplace") } /** @@ -1432,11 +1520,15 @@ trait OptimisticTransactionImpl extends TransactionHelper newMode = _newMetadata.columnMappingMode ) + val isBothColumnMappingEnabled = + _newMetadata.columnMappingMode != NoMapping && + _currentMetadata.columnMappingMode != NoMapping + def dropColumnOp: Boolean = DeltaColumnMapping.isDropColumnOperation( - _newMetadata, _currentMetadata) + _newMetadata.schema, _currentMetadata.schema, isBothColumnMappingEnabled) def renameColumnOp: Boolean = DeltaColumnMapping.isRenameColumnOperation( - _newMetadata, _currentMetadata) + _newMetadata.schema, _currentMetadata.schema, isBothColumnMappingEnabled) def columnMappingChange: Boolean = isColumnMappingUpgrade || dropColumnOp || renameColumnOp @@ -1920,6 +2012,15 @@ trait OptimisticTransactionImpl extends TransactionHelper newProtocol.toIterator allActions = allActions.map { action => action match { + case dm: DomainMetadata if isClusteringChangedOnUCManagedTable(dm) => + // Temporary: block clustering changes on UC-managed tables (commitLarge() path). + // commitLarge() bypasses prepareCommit(), so this guard is needed separately. + // The check is intentionally inside the lazy map: commitLarge streams actions to + // avoid materialising large sets, so an eager pre-scan is not practical. The + // exception is thrown before any data is written to the commit coordinator because + // the iterator is consumed first during serialisation. + throw DeltaErrors.operationNotSupportedException( + "Clustering column changes on Unity Catalog managed tables") case a: AddFile => assertDeletionVectorWellFormed(a) validateAddFileInvariants(a, notNullPartitionCols) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/SnapshotManagement.scala b/spark/src/main/scala/org/apache/spark/sql/delta/SnapshotManagement.scala index cc23282fa32..77c208dfd7b 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/SnapshotManagement.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/SnapshotManagement.scala @@ -1407,13 +1407,19 @@ trait SnapshotManagement { self: DeltaLog => // NOTE: Validation is a no-op with incremental commit disabled. newSnapshot.validateChecksum(Map("context" -> checksumContext)) } catch { - case _: IllegalStateException if !DeltaUtils.isTesting => false + case e: IllegalStateException if !DeltaUtils.isTesting => + logWarning(log"Incremental checksum validation failed: " + + log"${MDC(DeltaLogKeys.ERROR, e.getMessage)}") + false } if (!crcIsValid) { // Create snapshot without incremental checksum. This will fallback to creating // a checksum based on state reconstruction. Disable incremental commit to avoid // further error triggers in this session. + logWarning(log"Disabling incremental commit for this session due to checksum " + + log"validation failure at version " + + log"${MDC(DeltaLogKeys.VERSION, newSnapshot.version)}") spark.sessionState.conf.setConf(DeltaSQLConf.INCREMENTAL_COMMIT_ENABLED, false) spark.sessionState.conf.setConf(DeltaSQLConf.DELTA_WRITE_SET_TRANSACTIONS_IN_CRC, false) return createSnapshotWithCrc(checksumOpt = None) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/catalog/AbstractDeltaCatalog.scala b/spark/src/main/scala/org/apache/spark/sql/delta/catalog/AbstractDeltaCatalog.scala index c6e9d07e329..c49648db277 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/catalog/AbstractDeltaCatalog.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/catalog/AbstractDeltaCatalog.scala @@ -155,10 +155,20 @@ class AbstractDeltaCatalog extends DelegatingCatalogExtension Option(allTableProperties.get("location")) } val id = { - TableIdentifier(ident.name(), ident.namespace().lastOption) + // Preserve the catalog name in the V1 identifier for Unity Catalog because this `id` + // becomes `tableDesc.identifier` for the rest of the create/replace flow. Downstream + // catalog-update logic reads `table.identifier.catalog`; without it, the table looks like + // an unqualified V1 table and catalog updates route through the current/session catalog + // instead of the delegated Unity Catalog entry. + val base = TableIdentifier(ident.name(), ident.namespace().lastOption) + if (isUnityCatalog) { + base.copy(catalog = Some(name())) // `name()` here is the catalog name. + } else { + base + } } var locUriOpt = location.map(CatalogUtils.stringToURI) - val existingTableOpt = getExistingTableIfExists(id) + val existingTableOpt = getExistingTableIfExists(id, Some(ident), operation) // PROP_IS_MANAGED_LOCATION indicates that the table location is not user-specified but // system-generated. The table should be created as managed table in this case. val isManagedLocation = Option(allTableProperties.get(TableCatalog.PROP_IS_MANAGED_LOCATION)) @@ -558,8 +568,19 @@ class AbstractDeltaCatalog extends DelegatingCatalogExtension properties = validatedConfigurations) } - /** Checks if a table already exists for the provided identifier. */ - def getExistingTableIfExists(table: TableIdentifier): Option[CatalogTable] = { + /** + * Checks if a Delta table already exists for the provided identifier. + * + * This first goes through the legacy V1 SessionCatalog/HMS lookup using [[TableIdentifier]]. + * For operations that target an existing Unity Catalog table, it then falls back to the + * delegated V2 catalog plugin lookup (for example Unity Catalog) when the V1 lookup does not + * surface the table entry. + */ + def getExistingTableIfExists( + table: TableIdentifier, + identOpt: Option[Identifier], + operation: TableCreationModes.CreationMode) + : Option[CatalogTable] = { // If this is a path identifier, we cannot return an existing CatalogTable. The Create command // will check the file system itself if (isPathIdentifier(table)) return None @@ -573,6 +594,50 @@ class AbstractDeltaCatalog extends DelegatingCatalogExtension throw DeltaErrors.notADeltaTable(table.table) } Some(oldTable) + } else if (operation != TableCreationModes.Create) { + identOpt match { + case Some(ident) => getExistingTableFromDelegatedCatalog(ident) + case None => + logDebug(log"Delegated catalog lookup skipped because no V2 identifier was provided " + + log"for ${MDC(DeltaLogKeys.TABLE_NAME, table)} during ${MDC(DeltaLogKeys.OPERATION, + operation.toString)}.") + None + } + } else { + None + } + } + + /** + * Returns an existing Delta table by querying the delegated catalog for Unity Catalog paths + * where the V1 SessionCatalog lookup does not surface the existing table entry. + * + * [[getExistingTableIfExists]] first checks the V1 catalog using a [[TableIdentifier]]. For + * Unity Catalog, some staged create/replace paths need a delegated V2 catalog lookup on the + * original [[Identifier]] to recover the existing table metadata. + * + * Even though this goes through the delegated V2 catalog plugin, Spark can still surface the + * result as a [[V1Table]] wrapper when the delegated catalog is exposing V1-backed table + * metadata. In that case we unwrap the embedded [[CatalogTable]] and continue on the V1 Delta + * path. + */ + private def getExistingTableFromDelegatedCatalog(ident: Identifier): Option[CatalogTable] = { + if (isUnityCatalog) { + try { + super.loadTable(ident) match { + case v1: V1Table if DeltaTableUtils.isDeltaTable(v1.catalogTable) => + Some(v1.catalogTable) + case _ => + logDebug(log"Delegated catalog lookup for ${MDC(DeltaLogKeys.TABLE_NAME, ident)} " + + log"did not return a Delta table.") + None + } + } catch { + case _: NoSuchTableException => + logDebug(log"Delegated catalog lookup did not find an existing table for " + + log"${MDC(DeltaLogKeys.TABLE_NAME, ident)}.") + None + } } else { None } @@ -617,11 +682,11 @@ class AbstractDeltaCatalog extends DelegatingCatalogExtension } /** - * The UC table ID property was renamed from an old name. In a transition period we need to - * translate the old UC table ID property name set by caller to new one. And in case both the new - * and old properties are set, remove the old one. Later in UC server it might throw error if it - * sees both. - * TODO: clean up once callers are migrated. + * Normalizes the deprecated UC table ID property key to the canonical key. + * + * This is temporary compatibility for callers that still send the old key during the rename + * transition. If both keys are present, we drop the old one and keep the canonical key only. + * TODO(issue #6296): remove once all callers stop sending the deprecated key. */ private def translateUCTableIdProperty(props: util.Map[String, String]): Unit = { val oldTableIdProperty = Option(props.remove(UC_TABLE_ID_KEY_OLD)) @@ -654,6 +719,11 @@ class AbstractDeltaCatalog extends DelegatingCatalogExtension writeOptions = sqlWriteOptions } expandTableProps(props, writeOptions, conf) + if (isUnityCatalog) { + // Unity Catalog callers may still send the deprecated `ucTableId` property key. + // Normalize it here to the canonical `io.unitycatalog.tableId` key before create/replace. + translateUCTableIdProperty(props) + } createDeltaTable( ident, schema, diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/catalog/DeltaTableV2.scala b/spark/src/main/scala/org/apache/spark/sql/delta/catalog/DeltaTableV2.scala index a980d8983d1..bd4bb302fc8 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/catalog/DeltaTableV2.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/catalog/DeltaTableV2.scala @@ -242,7 +242,9 @@ class DeltaTableV2 private( base.put(TableCatalog.PROP_OWNER, table.owner) } v1Table.storage.properties.foreach { case (key, value) => - base.put(TableCatalog.OPTION_PREFIX + key, value) + if (!DeltaTableV2.HIDDEN_STORAGE_PROPERTY_PREFIXES.exists(key.startsWith)) { + base.put(TableCatalog.OPTION_PREFIX + key, value) + } } if (v1Table.tableType == CatalogTableType.EXTERNAL) { base.put(TableCatalog.PROP_EXTERNAL, "true") @@ -434,6 +436,15 @@ class DeltaTableV2 private( } object DeltaTableV2 { + + /** + * Storage property key prefixes that should be excluded from the user-visible V2 table + * properties returned by [[DeltaTableV2.properties()]]. These are Hadoop filesystem + * configuration options that may contain sensitive credentials (access keys, session + * tokens, etc.) injected by catalogs at table-load time. + */ + private[delta] val HIDDEN_STORAGE_PROPERTY_PREFIXES: Seq[String] = Seq("fs.") + def unapply(deltaTable: DeltaTableV2): Option[( SparkSession, Path, diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/CreateDeltaTableCommand.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/CreateDeltaTableCommand.scala index b471f787225..f23fd6937e7 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/CreateDeltaTableCommand.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/CreateDeltaTableCommand.scala @@ -786,6 +786,11 @@ case class CreateDeltaTableCommand( newMetadata.configuration, txn.snapshot) newMetadata = newMetadata.copy(configuration = updatedConfig) + if (allowCatalogManaged && txn.snapshot.isCatalogOwned) { + // Preserve the existing Delta metadata id across REPLACE. This is distinct from the + // Unity Catalog table id stored in `io.unitycatalog.tableId`. + newMetadata = newMetadata.copy(id = txn.snapshot.metadata.id) + } txn.updateMetadataForNewTableInReplace(newMetadata) } } @@ -802,7 +807,7 @@ case class CreateDeltaTableCommand( deltaLog: DeltaLog, tableWithLocation: CatalogTable, snapshotOpt: Option[Snapshot] = None): OptimisticTransaction = { - val txn = deltaLog.startTransaction(None, snapshotOpt) + val txn = deltaLog.startTransaction(existingTableOpt, snapshotOpt) validatePrerequisitesForClusteredTable(txn.snapshot.protocol, txn.deltaLog) // During CREATE (not REPLACE/overwrites), we synchronously run conversion diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/CreateDeltaTableLike.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/CreateDeltaTableLike.scala index 04913683e94..d0e38f95e65 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/CreateDeltaTableLike.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/CreateDeltaTableLike.scala @@ -24,6 +24,7 @@ import org.apache.spark.sql.{SaveMode, SparkSession} import org.apache.spark.sql.catalyst.SQLConfHelper import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogTableType} import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.catalyst.util.CharVarcharUtils import org.apache.spark.sql.connector.catalog.Identifier import org.apache.spark.sql.types.StructType @@ -106,15 +107,26 @@ trait CreateDeltaTableLike extends SQLConfHelper { } case TableCreationModes.Replace | TableCreationModes.CreateOrReplace if existingTableOpt.isDefined => - UpdateCatalogFactory.getUpdateCatalogHook(table, spark).updateSchema(spark, snapshot) + // Catalog-managed / CC tables are owned by the delegated V2 catalog plugin (for example + // Unity Catalog), so SessionCatalog's post-commit UpdateCatalogHook must not run. + if (!allowCatalogManaged) { + UpdateCatalogFactory.getUpdateCatalogHook(table, spark).updateSchema(spark, snapshot) + } case TableCreationModes.Replace => val ident = Identifier.of(table.identifier.database.toArray, table.identifier.table) throw DeltaErrors.cannotReplaceMissingTableException(ident) case TableCreationModes.CreateOrReplace => - spark.sessionState.catalog.createTable( - cleaned, - ignoreIfExists = false, - validateLocation = false) + createTableFunc match { + case Some(createFunc) => + // This is the new missing-table path where creation is delegated through the V2 + // catalog plugin (for example Unity Catalog) instead of SessionCatalog.createTable(). + createFunc(cleaned) + case None => + spark.sessionState.catalog.createTable( + cleaned, + ignoreIfExists = false, + validateLocation = false) + } } if (conf.getConf(DeltaSQLConf.HMS_FORCE_ALTER_TABLE_DATA_SCHEMA)) { spark.sessionState.catalog.alterTableDataSchema(cleaned.identifier, cleaned.schema) @@ -152,13 +164,23 @@ trait CreateDeltaTableLike extends SQLConfHelper { ++ additionalProperties, storage = storageProps, tracksPartitionsInCatalog = true) - } else { + } else if (allowCatalogManaged) { // Setting table properties is required for creating catalogManaged tables. - val properties: Map[String, String] = - if (allowCatalogManaged) UpdateCatalog.updatedProperties(snapshot) else Map.empty + table.copy( + // Here we use snapshot.schema instead of table.schema because it reflects the actual + // committed state of the table. + // Delta does not have a distinct storage type for Char/Varchar; in snapshots, they are + // represented in String type with extra type metadata. We convert them to back to the + // original Char/Varchar types when storing them in the catalog. + schema = CharVarcharUtils.getRawSchema(snapshot.schema), + partitionColumnNames = snapshot.metadata.partitionColumns, + properties = UpdateCatalog.updatedProperties(snapshot), + storage = storageProps, + tracksPartitionsInCatalog = true) + } else { table.copy( schema = new StructType(), - properties = properties, + properties = Map.empty, partitionColumnNames = Nil, // Remove write specific options when updating the catalog storage = storageProps, diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/commands/WriteIntoDelta.scala b/spark/src/main/scala/org/apache/spark/sql/delta/commands/WriteIntoDelta.scala index f2775c39cb8..bd6229eb262 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/commands/WriteIntoDelta.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/commands/WriteIntoDelta.scala @@ -333,11 +333,33 @@ case class WriteIntoDelta( txn, data, options ) val addFiles = newFiles.collect { case a: AddFile => a } + + if (!useDynamicPartitionOverwriteMode && + options.useNullIntolerantEqualityWithDPO.isDefined) { + throw DeltaErrors.illegalDeltaOptionException( + name = DeltaOptions.USE_NULL_INTOLERANT_EQUALITY_WITH_DPO, + input = options.useNullIntolerantEqualityWithDPO.get.toString, + explain = "This option should be specified only in Dynamic Partition Overwrite mode.") + } + val deletedFiles = if (useDynamicPartitionOverwriteMode) { // with dynamic partition overwrite for any partition that is being written to all // existing data in that partition will be deleted. - // the selection what to delete is on the next two lines - val updatePartitions = addFiles.map(_.partitionValues).toSet + // the selection what to delete is determined by `updatePartitions`. + + // Dynamic Partition Overwrite (DPO) uses null-tolerant equality, meaning NULL partitions + // in the table will be overwritten if there are matching NULL values in the query. + // This option simulates null-intolerant equality by not including partitions with + // NULL values in the set of partitions to be overwritten. + val updatePartitions = + if (options.useNullIntolerantEqualityWithDPO.contains(true)) { + addFiles.collect { case addFile + if addFile.partitionValues.forall { case (_, value) => value != null } + => addFile.partitionValues + }.toSet + } else { + addFiles.map(_.partitionValues).toSet + } txn.filterFiles(updatePartitions).map(_.remove) } else { txn.filterFiles().map(_.remove) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsUtils.scala b/spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsUtils.scala index 1e01bf95ae0..a6c2a26d7e7 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsUtils.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/CoordinatedCommitsUtils.scala @@ -26,6 +26,7 @@ import org.apache.spark.sql.delta.actions.{Metadata, Protocol, TableFeatureProto import org.apache.spark.sql.delta.commands.CloneTableCommand import org.apache.spark.sql.delta.logging.DeltaLogKeys import org.apache.spark.sql.delta.metering.DeltaLogging +import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.{FileNames, JsonUtils} import org.apache.spark.sql.delta.util.FileNames.{BackfilledDeltaFile, CompactedDeltaFile, DeltaFile, UnbackfilledDeltaFile} import io.delta.storage.LogStore @@ -166,7 +167,7 @@ object CatalogOwnedTableUtils extends DeltaLogging { case spark.sessionState.analyzer.CatalogAndIdentifier(catalog, _) => if (catalog.getClass.getName == UCCommitCoordinatorBuilder.UNITY_CATALOG_CONNECTOR_CLASS) { // UC is the current commit coordinator. - Some("unity-catalog") + Some(UCCommitCoordinatorBuilder.COORDINATOR_NAME) } else { // Other catalog (e.g., `spark_catalog`) is the commit coordinator. Some(catalog.name) @@ -565,14 +566,48 @@ object CoordinatedCommitsUtils extends DeltaLogging { case UnbackfilledDeltaFile(fileStatus, version, _) if version > maxVersionSeen => (fileStatus, version) } - // Check for a gap between listing and commit files in the logsegment - val gapListing = unbackfilledDeltas.headOption match { - case Some((_, version)) if maxVersionSeen + 1 < version => - listDeltas(maxVersionSeen + 1, Some(version)) - // no gap before - case _ => Iterator.empty + val backfillGapFixEnabled = SparkSession.active.sessionState.conf.getConf( + DeltaSQLConf.COMMIT_FILES_ITERATOR_BACKFILL_GAP_FIX_ENABLED) + if (backfillGapFixEnabled) { + // This fixes two bugs in the gap listing between Phase 1 (filesystem) and + // Phase 2 (coordinator/snapshot): + // + // Bug 1 - data loss: between Phase 1 and Phase 2, + // a concurrent writer backfills ALL remaining commits. unbackfilledDeltas is empty, + // headOption returns None, and the old code falls through to Iterator.empty, + // silently dropping versions [maxVersionSeen+1, endSnapshot.version]. + // Fix: fall back to filesystem listing up to endSnapshot.version. + // + // Bug 2 - duplicate entries (Some case): listDeltas uses an inclusive upper bound + // (takeWhile { version <= endVersion }). If the first unbackfilled version is also + // backfilled on filesystem (either it happened after we contacted UC, or UC did not + // know), the gap listing includes it, producing a duplicate with unbackfilledDeltas. + // Example: prev snapshot v100, latest v110. update() returns v105-v110 as + // unbackfilled, but v105 is backfilled on filesystem. Old code calls + // listDeltas(101, Some(105)), which includes 105.json. gapListing=[v101..v105], + // unbackfilledDeltas=[v105..v110]. v105 appears twice. + // Fix: use version - 1 as exclusive upper bound. listDeltas(101, Some(104)) stops + // before v105, eliminating the overlap. + val highestGapVersion = unbackfilledDeltas.headOption match { + case Some((_, version)) => version - 1 + case None => endSnapshot.version + } + val gapListing = if (maxVersionSeen < highestGapVersion) { + listDeltas(maxVersionSeen + 1, Some(highestGapVersion)) + } else { + Iterator.empty + } + gapListing ++ unbackfilledDeltas + } else { + // Check for a gap between listing and commit files in the logsegment + val gapListing = unbackfilledDeltas.headOption match { + case Some((_, version)) if maxVersionSeen + 1 < version => + listDeltas(maxVersionSeen + 1, Some(version)) + // no gap before + case _ => Iterator.empty + } + gapListing ++ unbackfilledDeltas } - gapListing ++ unbackfilledDeltas } // We want to avoid invoking `tailFromSnapshot()` as it internally calls deltaLog.update() @@ -771,6 +806,18 @@ object CoordinatedCommitsUtils extends DeltaLogging { properties.filter { case (k, _) => ICT_TABLE_PROPERTY_KEYS.contains(k) } } + /** + * Extracts the explicit QoL configurations from the provided properties. + * + * These are preserved across catalog-managed REPLACE when the existing table already has the + * QoL defaults materialized in metadata, so a no-op REPLACE does not accidentally drop them + * while rebuilding the configuration map. + */ + def getExplicitQoLConfigurations(properties: Map[String, String]): Map[String, String] = { + val qolKeys = CatalogOwnedTableUtils.QOL_TABLE_FEATURES_AND_PROPERTIES.map(_._2.key).toSet + properties.filter { case (k, _) => qolKeys.contains(k) } + } + /** * Fetches the SparkSession default configurations for ICT. The `withDefaultKey` * flag controls whether the keys in the returned map should have the default prefix or not. diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/UCCommitCoordinatorBuilder.scala b/spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/UCCommitCoordinatorBuilder.scala index 4dc34819946..0f92e15f870 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/UCCommitCoordinatorBuilder.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/coordinatedcommits/UCCommitCoordinatorBuilder.scala @@ -43,6 +43,9 @@ import org.apache.spark.sql.SparkSession object UCCommitCoordinatorBuilder extends CatalogOwnedCommitCoordinatorBuilder with DeltaLogging { + /** The coordinator name used in table metadata to identify UC-managed tables. */ + final val COORDINATOR_NAME: String = "unity-catalog" + /** Prefix for Spark SQL catalog configurations. */ final private val SPARK_SQL_CATALOG_PREFIX = "spark.sql.catalog." @@ -64,7 +67,7 @@ object UCCommitCoordinatorBuilder // Use a var instead of val for ease of testing by injecting different UCClientFactory. private[delta] var ucClientFactory: UCClientFactory = UCTokenBasedRestClientFactory - override def getName: String = "unity-catalog" + override def getName: String = COORDINATOR_NAME override def build(spark: SparkSession, conf: Map[String, String]): CommitCoordinatorClient = { val metastoreId = conf.getOrElse( diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/files/DelayedCommitProtocol.scala b/spark/src/main/scala/org/apache/spark/sql/delta/files/DelayedCommitProtocol.scala index 23df32bea14..881e8ffdcf8 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/files/DelayedCommitProtocol.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/files/DelayedCommitProtocol.scala @@ -109,22 +109,17 @@ class DelayedCommitProtocol( addedFiles = new ArrayBuffer[(Map[String, String], String)] } - /** Prefix added in testing mode to all filenames to test special chars that need URL-encoding. */ - val FILE_NAME_PREFIX = SQLConf.get.getConf(DeltaSQLConf.TEST_FILE_NAME_PREFIX) - + // The file name looks like part-r-00000-2dd664f9-d2c4-4ffe-878f-c6c70c1fb0cb_00003.gz.parquet + // Note that %05d does not truncate the split number, so if we have more than 100000 tasks, + // the file name is fine and won't overflow. + // CDC files (CDC_PARTITION_COL = true) are named with "cdc-..." instead of "part-...". protected def getFileName( taskContext: TaskAttemptContext, ext: String, partitionValues: Map[String, String]): String = { - // The file name looks like part-r-00000-2dd664f9-d2c4-4ffe-878f-c6c70c1fb0cb_00003.gz.parquet - // Note that %05d does not truncate the split number, so if we have more than 100000 tasks, - // the file name is fine and won't overflow. val split = taskContext.getTaskAttemptID.getTaskID.getId - val uuid = UUID.randomUUID.toString - // CDC files (CDC_PARTITION_COL = true) are named with "cdc-..." instead of "part-...". - val typePrefix = - if (partitionValues.get(CDC_PARTITION_COL).contains("true")) "cdc-" else "part-" - f"${FILE_NAME_PREFIX}${typePrefix}${split}%05d-${uuid}${ext}" + val isCdc = partitionValues.get(CDC_PARTITION_COL).contains("true") + DelayedCommitProtocol.buildFileName(split, ext, isCdc) } protected def parsePartitions( @@ -282,3 +277,23 @@ class DelayedCommitProtocol( // TODO: we can also try delete the addedFiles as a best-effort cleanup. } } + +object DelayedCommitProtocol { + /** + * Builds a Delta data file name following the standard naming convention: + * {testPrefix}{typePrefix}{splitId%05d}-{uuid}{ext} + * + * Shared between V1 (DelayedCommitProtocol.getFileName) and DSv2 (DataWriter). + * + * @param splitId the task split/partition ID + * @param ext file extension including dot (e.g. ".snappy.parquet") + * @param isCdc true for CDC files (uses "cdc-" prefix instead of "part-") + * @return the file name string + */ + def buildFileName(splitId: Int, ext: String, isCdc: Boolean): String = { + val fileNamePrefix = SQLConf.get.getConf(DeltaSQLConf.TEST_FILE_NAME_PREFIX) + val uuid = UUID.randomUUID.toString + val typePrefix = if (isCdc) "cdc-" else "part-" + f"${fileNamePrefix}${typePrefix}${splitId}%05d-${uuid}${ext}" + } +} diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/fuzzer/OptimisticTransactionPhases.scala b/spark/src/main/scala/org/apache/spark/sql/delta/fuzzer/OptimisticTransactionPhases.scala index 4fc42e666ce..0c452183afb 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/fuzzer/OptimisticTransactionPhases.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/fuzzer/OptimisticTransactionPhases.scala @@ -17,6 +17,7 @@ package org.apache.spark.sql.delta.fuzzer case class OptimisticTransactionPhases( + analysisPhase: ExecutionPhaseLock, initialPhase: ExecutionPhaseLock, preparePhase: ExecutionPhaseLock, commitPhase: ExecutionPhaseLock, @@ -27,6 +28,7 @@ object OptimisticTransactionPhases { private final val PREFIX = "TXN_" + final val ANALYSIS_PHASE_LABEL = PREFIX + "ANALYSIS" final val INITIAL_PHASE_LABEL = PREFIX + "INIT" final val PREPARE_PHASE_LABEL = PREFIX + "PREPARE" final val COMMIT_PHASE_LABEL = PREFIX + "COMMIT" @@ -39,6 +41,7 @@ object OptimisticTransactionPhases { txnName + "-" + phaseLabel OptimisticTransactionPhases( + analysisPhase = ExecutionPhaseLock(toTxnPhaseLabel(ANALYSIS_PHASE_LABEL)), initialPhase = ExecutionPhaseLock(toTxnPhaseLabel(INITIAL_PHASE_LABEL)), preparePhase = ExecutionPhaseLock(toTxnPhaseLabel(PREPARE_PHASE_LABEL)), commitPhase = ExecutionPhaseLock(toTxnPhaseLabel(COMMIT_PHASE_LABEL)), diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/fuzzer/PhaseLockingTransactionExecutionObserver.scala b/spark/src/main/scala/org/apache/spark/sql/delta/fuzzer/PhaseLockingTransactionExecutionObserver.scala index 62110fc0634..9b2d71502db 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/fuzzer/PhaseLockingTransactionExecutionObserver.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/fuzzer/PhaseLockingTransactionExecutionObserver.scala @@ -24,6 +24,7 @@ private[delta] class PhaseLockingTransactionExecutionObserver( with PhaseLockingExecutionObserver { override val phaseLocks: Seq[ExecutionPhaseLock] = Seq( + phases.analysisPhase, phases.initialPhase, phases.preparePhase, phases.commitPhase, @@ -46,8 +47,10 @@ private[delta] class PhaseLockingTransactionExecutionObserver( */ @volatile protected var autoAdvanceNextObserver: Boolean = false - override def startingTransaction(f: => OptimisticTransaction): OptimisticTransaction = + override def startingTransaction(f: => OptimisticTransaction): OptimisticTransaction = { + phases.analysisPhase.leave() phases.initialPhase.execute(f) + } override def preparingCommit[T](f: => T): T = phases.preparePhase.execute(f) diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/hooks/metrics/UpdateMetricsHook.scala b/spark/src/main/scala/org/apache/spark/sql/delta/hooks/metrics/UpdateMetricsHook.scala new file mode 100644 index 00000000000..07a259ee8f1 --- /dev/null +++ b/spark/src/main/scala/org/apache/spark/sql/delta/hooks/metrics/UpdateMetricsHook.scala @@ -0,0 +1,209 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.hooks.metrics + +// scalastyle:off import.ordering.noEmptyLine +import scala.collection.JavaConverters._ + +import com.fasterxml.jackson.annotation.JsonProperty +import io.delta.storage.commit.uccommitcoordinator.UCCommitCoordinatorClient + +import org.apache.spark.sql.delta.CommittedTransaction +import org.apache.spark.sql.delta.actions.Action +import org.apache.spark.sql.delta.coordinatedcommits.UCCommitCoordinatorBuilder +import org.apache.spark.sql.delta.hooks.PostCommitHook +import org.apache.spark.sql.delta.logging.DeltaLogKeys +import org.apache.spark.sql.delta.metering.DeltaLogging +import org.apache.spark.sql.delta.util.{CatalogTableUtils, JsonUtils} + +import io.unitycatalog.client.auth.TokenProvider +import org.apache.http.HttpHeaders +import org.apache.http.client.config.RequestConfig +import org.apache.http.client.methods.HttpPost +import org.apache.http.entity.{ContentType, StringEntity} +import org.apache.http.impl.client.{CloseableHttpClient, HttpClientBuilder} +import org.apache.http.util.EntityUtils +import org.apache.spark.internal.MDC +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.catalog.CatalogTable + +// Payload case classes (JSON-serialized via Jackson @JsonProperty) + +case class ReportDeltaMetricsRequest( + @JsonProperty("table_id") tableId: String, + @JsonProperty("report") report: CommitReportEnvelope) + +case class CommitReportEnvelope( + @JsonProperty("commit_report") commitReport: CommitReport) + +case class CommitReport() + +/** + * Post-commit hook that sends commit metrics to Unity Catalog + * for UC-managed Delta tables. + * + * Best-effort: failures are logged as warnings and never fail + * the commit. + * + * @param catalogTable catalog metadata for UC-managed detection + */ +case class UpdateMetricsHook(catalogTable: Option[CatalogTable]) + extends PostCommitHook with DeltaLogging { + + override val name: String = "Update catalog metrics" + + override def run( + spark: SparkSession, + txn: CommittedTransaction): Unit = { + val ct = catalogTable.orNull + if (ct == null || + !CatalogTableUtils.isUnityCatalogManagedTable(ct)) { + return + } + val tableId = ct.storage.properties( + UCCommitCoordinatorClient.UC_TABLE_ID_KEY) + + try { + val request = UpdateMetricsHook.buildRequest( + tableId, txn.committedActions, txn.committedVersion) + val catalogName = ct.identifier.catalog + UpdateMetricsHook.sendMetrics( + spark, request, catalogName = catalogName) + + logDebug( + log"Successfully sent UC metrics for table " + + log"${MDC(DeltaLogKeys.PATH, txn.deltaLog.logPath)} " + + log"version " + + log"${MDC(DeltaLogKeys.VERSION, txn.committedVersion)}") + + } catch { + case e: Exception => + logWarning( + log"Failed to send UC metrics for table " + + log"${MDC(DeltaLogKeys.PATH, txn.deltaLog.logPath)} " + + log"version " + + log"${MDC(DeltaLogKeys.VERSION, txn.committedVersion)}" + + log": ${MDC(DeltaLogKeys.ERROR, e.getMessage)}", e) + } + } +} + +object UpdateMetricsHook { + + // -- Payload builder -- + + private[metrics] def buildRequest( + tableId: String, + _committedActions: Seq[Action], + _committedVersion: Long): ReportDeltaMetricsRequest = { + ReportDeltaMetricsRequest( + tableId = tableId, + report = CommitReportEnvelope(CommitReport()) + ) + } + + // -- HTTP client -- + + private val CATALOG_CONF_PREFIX = "spark.sql.catalog" + private val CATALOG_URI_CONF_SUFFIX = "uri" + private val METRICS_ENDPOINT_PATH = + "/api/2.1/unity-catalog/delta/preview/metrics" + private val HTTP_TIMEOUT_MS = 5000L + + private[metrics] def sendMetrics( + spark: SparkSession, + request: ReportDeltaMetricsRequest, + catalogName: Option[String] = None): Unit = { + val catalog = catalogName.getOrElse( + throw new IllegalArgumentException( + "Catalog name required for UC metrics; " + + "endpoint URI is read from " + + "spark.sql.catalog..uri")) + val endpointUrl = getEndpointUrl(spark, catalog) + val authToken = getAuthToken(spark, catalog) + + val requestConfig = RequestConfig.custom() + .setConnectTimeout(HTTP_TIMEOUT_MS.toInt) + .setSocketTimeout(HTTP_TIMEOUT_MS.toInt) + .setConnectionRequestTimeout(HTTP_TIMEOUT_MS.toInt) + .build() + + val httpClient: CloseableHttpClient = + HttpClientBuilder.create() + .setDefaultRequestConfig(requestConfig) + .build() + + try { + val httpPost = new HttpPost(endpointUrl) + httpPost.setHeader( + HttpHeaders.AUTHORIZATION, s"Bearer $authToken") + + val jsonPayload = JsonUtils.toJson(request) + httpPost.setEntity( + new StringEntity( + jsonPayload, ContentType.APPLICATION_JSON)) + + val response = httpClient.execute(httpPost) + try { + val statusCode = response.getStatusLine.getStatusCode + if (statusCode < 200 || statusCode >= 300) { + val responseBody = Option(response.getEntity) + .map(EntityUtils.toString) + .getOrElse("") + throw new RuntimeException( + "UC metrics endpoint returned error " + + s"status $statusCode: $responseBody") + } + } finally { + response.close() + } + } finally { + httpClient.close() + } + } + + private def getEndpointUrl( + spark: SparkSession, + catalogName: String): String = { + val uriKey = s"$CATALOG_CONF_PREFIX.$catalogName" + + s".$CATALOG_URI_CONF_SUFFIX" + spark.conf.getOption(uriKey) match { + case Some(uri) if uri.nonEmpty => + s"${uri.stripSuffix("/")}$METRICS_ENDPOINT_PATH" + case _ => + throw new IllegalArgumentException( + s"UC catalog base URI not configured. " + + s"Set $uriKey") + } + } + + private def getAuthToken( + spark: SparkSession, + catalogName: String): String = { + val configMap = + UCCommitCoordinatorBuilder.getCatalogConfigMap(spark) + val config = configMap.get(catalogName).getOrElse( + throw new IllegalArgumentException( + "Unity Catalog configuration not found for " + + s"catalog '$catalogName'. Configure " + + "spark.sql.catalog..uri and auth " + + "(auth.type/auth.token or legacy .token).")) + val tokenProvider = + TokenProvider.create(config.authConfig.asJava) + tokenProvider.accessToken() + } +} diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/schema/SchemaUtils.scala b/spark/src/main/scala/org/apache/spark/sql/delta/schema/SchemaUtils.scala index d5f443c7a20..5b4937cd9ff 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/schema/SchemaUtils.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/schema/SchemaUtils.scala @@ -109,6 +109,24 @@ object SchemaUtils extends DeltaLogging { Some(other).filter(f) } + /** + * Checks if a given data type contains a NullType, including inside UDTs. + * `typeExistsRecursively` does not recurse into UDT sqlTypes, so this method + * explicitly handles that case. + */ + def nullTypeExistsRecursively( + t: DataType + ): Boolean = { + typeExistsRecursively(t) { + case _: NullType => + true + case udt: UserDefinedType[_] => + nullTypeExistsRecursively(udt.sqlType) + case _ => + false + } + } + /** Turns the data types to nullable in a recursive manner for nested columns. */ def typeAsNullable(dt: DataType): DataType = dt match { case s: StructType => s.asNullable @@ -130,7 +148,8 @@ object SchemaUtils extends DeltaLogging { */ def dropNullTypeColumns(df: DataFrame): DataFrame = { val schema = df.schema - if (!typeExistsRecursively(schema)(_.isInstanceOf[NullType])) return df + if (!nullTypeExistsRecursively(schema)) return df + def generateSelectExpr(sf: StructField, nameStack: Seq[String]): Column = sf.dataType match { case st: StructType => val nested = st.fields.flatMap { f => @@ -144,16 +163,21 @@ object SchemaUtils extends DeltaLogging { when(col(colName).isNull, null) .otherwise(struct(nested: _*)) .alias(sf.name) - case a: ArrayType if typeExistsRecursively(a)(_.isInstanceOf[NullType]) => + case a: ArrayType if nullTypeExistsRecursively(a) => val colName = UnresolvedAttribute.apply(nameStack :+ sf.name).sql throw new DeltaAnalysisException( errorClass = "DELTA_COMPLEX_TYPE_COLUMN_CONTAINS_NULL_TYPE", messageParameters = Array(colName, "ArrayType")) - case m: MapType if typeExistsRecursively(m)(_.isInstanceOf[NullType]) => + case m: MapType if nullTypeExistsRecursively(m) => val colName = UnresolvedAttribute.apply(nameStack :+ sf.name).sql throw new DeltaAnalysisException( errorClass = "DELTA_COMPLEX_TYPE_COLUMN_CONTAINS_NULL_TYPE", - messageParameters = Array(colName, "NullType")) + messageParameters = Array(colName, "MapType")) + case udt: UserDefinedType[_] if nullTypeExistsRecursively(udt.sqlType) => + val colName = UnresolvedAttribute.apply(nameStack :+ sf.name).sql + throw new DeltaAnalysisException( + errorClass = "DELTA_USER_DEFINED_TYPE_COLUMN_CONTAINS_NULL_TYPE", + messageParameters = Array(colName, udt.userClass.getName)) case _ => val colName = UnresolvedAttribute.apply(nameStack :+ sf.name).sql col(colName).alias(sf.name) @@ -263,7 +287,7 @@ object SchemaUtils extends DeltaLogging { return nullFields.headOption } - if (typeExistsRecursively(schema)(_.isInstanceOf[NullType])) { + if (nullTypeExistsRecursively(schema)) { findNullTypeColumnRec(schema, Seq.empty) } else { None diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlannedTable.scala b/spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlannedTable.scala index 54a449dea9b..e6e0057fcaf 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlannedTable.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlannedTable.scala @@ -179,7 +179,7 @@ class ServerSidePlannedTable( tableName: String, tableSchema: StructType, planningClient: ServerSidePlanningClient) - extends Table with SupportsRead with DeltaLogging { + extends Table with SupportsRead with AutoCloseable with DeltaLogging { // Returns fully qualified name (e.g., "catalog.database.table"). // The databaseName parameter receives ident.namespace().mkString(".") from DeltaCatalog, @@ -195,6 +195,10 @@ class ServerSidePlannedTable( override def newScanBuilder(options: CaseInsensitiveStringMap): ScanBuilder = { new ServerSidePlannedScanBuilder(spark, databaseName, tableName, tableSchema, planningClient) } + + override def close(): Unit = { + planningClient.close() + } } /** @@ -341,13 +345,19 @@ class ServerSidePlannedScan( } } - // Call the server-side planning API to get the scan plan with files AND credentials - private lazy val scanPlan: ScanPlan = planningClient.planScan( - databaseName, - tableName, - combinedFilter, - projectionColumnNames, - limit) + // Call the server-side planning API to get the scan plan with files AND credentials. + // Close the client after planning - the scan plan contains all data needed for partition + // creation and reading, so the client (and its HTTP connection) is no longer needed. + private lazy val scanPlan: ScanPlan = { + val plan = planningClient.planScan( + databaseName, + tableName, + combinedFilter, + projectionColumnNames, + limit) + planningClient.close() + plan + } // Explicitly signal that columnar is unsupported to prevent early enumeration of the partitions override def columnarSupportMode(): Scan.ColumnarSupportMode = diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlanningClient.scala b/spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlanningClient.scala index 2ade4f0ba8f..2162e9b8b78 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlanningClient.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlanningClient.scala @@ -35,7 +35,7 @@ case class ScanFile( * representation for filter pushdown. This keeps the interface catalog-agnostic while allowing * each server-side planning catalog implementation to convert filters to their own native format. */ -trait ServerSidePlanningClient { +trait ServerSidePlanningClient extends AutoCloseable { /** * Plan a table scan and return the list of files to read. * @@ -61,6 +61,12 @@ trait ServerSidePlanningClient { * @return true if ALL filters can be converted, false if ANY filter cannot be converted */ def canConvertFilters(filters: Array[Filter]): Boolean + + /** + * Close any resources held by this client. + * Default implementation is a no-op for clients that don't hold resources. + */ + override def close(): Unit = {} } /** diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlanningMetadata.scala b/spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlanningMetadata.scala index a1b8e4c889d..ab7905b4577 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlanningMetadata.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlanningMetadata.scala @@ -32,9 +32,11 @@ private[serverSidePlanning] trait ServerSidePlanningMetadata { def planningEndpointUri: String /** - * Authentication token for the planning endpoint. + * Supplier of authentication tokens for the planning endpoint. + * Called per-request to get the current token (supports OAuth). + * Returns None if no authentication is configured. */ - def authToken: Option[String] + def tokenSupplier: Option[() => String] /** * Catalog name for configuration lookups. @@ -56,7 +58,7 @@ private[serverSidePlanning] case class DefaultMetadata( catalogName: String, tableProps: Map[String, String] = Map.empty) extends ServerSidePlanningMetadata { override def planningEndpointUri: String = "" - override def authToken: Option[String] = None + override def tokenSupplier: Option[() => String] = None override def tableProperties: Map[String, String] = tableProps } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/UnityCatalogMetadata.scala b/spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/UnityCatalogMetadata.scala index 7406bf4edd9..04c175111c8 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/UnityCatalogMetadata.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/UnityCatalogMetadata.scala @@ -16,29 +16,28 @@ package org.apache.spark.sql.delta.serverSidePlanning +import scala.jdk.CollectionConverters._ + +import io.unitycatalog.client.auth.TokenProvider + import org.apache.spark.sql.SparkSession import org.apache.spark.sql.connector.catalog.{Identifier, Table} /** * Metadata for Unity Catalog tables. - * Provides base Iceberg REST endpoint for server-side planning. + * Provides base Iceberg REST endpoint and authentication for server-side planning. */ case class UnityCatalogMetadata( catalogName: String, ucUri: String, - ucToken: String, + override val tokenSupplier: Option[() => String], tableProps: Map[String, String]) extends ServerSidePlanningMetadata { override def planningEndpointUri: String = { - // Return base Iceberg REST path up to /v1/ - // The IcebergRESTCatalogPlanningClient will call /v1/config to get the prefix - // and construct the full URL according to the Iceberg REST catalog spec val base = if (ucUri.endsWith("/")) ucUri.dropRight(1) else ucUri s"$base/api/2.1/unity-catalog/iceberg-rest/v1" } - override def authToken: Option[String] = Some(ucToken) - override def tableProperties: Map[String, String] = tableProps } @@ -58,11 +57,55 @@ object UnityCatalogMetadata { // Read UC configuration from Spark conf val ucUri = spark.conf.get(s"spark.sql.catalog.$catalogName.uri", "") - val ucToken = spark.conf.get(s"spark.sql.catalog.$catalogName.token", "") + val authConfig = extractAuthConfig(spark, catalogName) + + val supplier: Option[() => String] = if (authConfig.nonEmpty) { + val tp = TokenProvider.create(authConfig.asJava) + Some(() => tp.accessToken()) + } else { + None + } - // Table properties currently unused, may be needed in future val tableProps = Map.empty[String, String] + UnityCatalogMetadata(catalogName, ucUri, supplier, tableProps) + } - UnityCatalogMetadata(catalogName, ucUri, ucToken, tableProps) + /** + * Extract authentication config for the planning endpoint. + * + * Follows the same pattern as UC's AuthConfigUtils: + * 1. Scan spark.sql.catalog..auth.* keys, strip prefix + * 2. If legacy .token exists: + * - Error if auth.token also set (configured twice) + * - Otherwise convert to Map("type" -> "static", ...) + * 3. Result passed to TokenProvider.create() + */ + private[serverSidePlanning] def extractAuthConfig( + spark: SparkSession, + catalogName: String): Map[String, String] = { + val catalogPrefix = s"spark.sql.catalog.$catalogName." + val authPrefix = s"${catalogPrefix}auth." + + // Extract all auth.* configs, stripping the prefix + val newFormatConfig = spark.conf.getAll + .filterKeys(_.startsWith(authPrefix)) + .map { case (fullKey, value) => (fullKey.stripPrefix(authPrefix), value) } + .toMap + + // Check legacy token + val legacyToken = spark.conf.get(s"${catalogPrefix}token", "") + + if (legacyToken.nonEmpty) { + // Conflict: both legacy .token and auth.token set + require(!newFormatConfig.contains("token"), + s"Static token configured twice. Use either " + + s"'spark.sql.catalog.$catalogName.token' (legacy) or " + + s"'spark.sql.catalog.$catalogName.auth.token' (new), not both.") + + // Legacy token: convert to static format. Matches UC AuthConfigUtils behavior. + Map("type" -> "static", "token" -> legacyToken) + } else { + newFormatConfig + } } } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala index b57dca07129..f86266fb1da 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala @@ -470,6 +470,16 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { .booleanConf .createWithDefault(true) + val DELTA_ALWAYS_COLLECT_STATS = + buildConf("alwaysCollectStats.enabled") + .internal() + .doc("When true, row counts are collected from file statistics even when there are no " + + "data filters. This is useful for ensuring PreparedDeltaFileIndex always has row count " + + "information available. Note: this may have a small performance overhead as it requires " + + "summing numRecords from all files.") + .booleanConf + .createWithDefault(false) + val DELTA_LIMIT_PUSHDOWN_ENABLED = buildConf("stats.limitPushdown.enabled") .internal() @@ -742,7 +752,7 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { |and streaming inserts with struct type cast. |""".stripMargin) .booleanConf - .createWithDefault(DeltaUtils.isTesting) + .createWithDefault(true) val DELTA_MERGE_PRESERVE_NULL_SOURCE_STRUCTS_UPDATE_STAR = buildConf("merge.preserveNullSourceStructs.updateStar") @@ -765,7 +775,19 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { |type cast during INSERT operations. |""".stripMargin) .booleanConf - .createWithDefault(DeltaUtils.isTesting) + .createWithDefault(true) + + val DELTA_INSERT_BY_NAME_SCHEMA_EVOLUTION_ENABLED = + buildConf("insert.byName.schemaEvolution.enabled") + .internal() + .doc( + """When enabled, SQL INSERT INTO BY NAME operations allow schema evolution: extra columns in + |the source that are not in the target table schema are added to the target schema when + |schema evolution (mergeSchema) is also enabled. Disable this flag to revert to the old + |behavior where extra columns always cause an error, regardless of schema evolution + |settings.""".stripMargin) + .booleanConf + .createWithDefault(true) val DELTA_SCHEMA_TYPE_CHECK = buildConf("schema.typeCheck.enabled") @@ -887,6 +909,15 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { .intConf .createWithDefault(5) + val COMMIT_FILES_ITERATOR_BACKFILL_GAP_FIX_ENABLED = + buildConf("coordinatedCommits.commitFilesIterator.backfillGapFix.enabled") + .internal() + .doc("When enabled, commitFilesIterator falls back to filesystem listing when all " + + "unbackfilled commits are concurrently backfilled between Phase 1 (filesystem listing) " + + "and Phase 2 (coordinator query), preventing silent data loss.") + .booleanConf + .createWithDefault(true) + ////////////////////////////////////////////// // DynamoDB Commit Coordinator-specific configs end ///////////////////////////////////////////// @@ -1355,6 +1386,15 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { .booleanConf .createWithDefault(true) + val DELTA_CHECKSUM_HISTOGRAM_FIELD_FOLLOWS_PROTOCOL = + buildConf("writeChecksumFile.histogramFollowsProtocol") + .internal() + .doc("""When true, writes the file size histogram to CRC files using the Delta spec field + |name "fileSizeHistogram". When false, uses the legacy Delta-Spark field name + |"histogramOpt".""".stripMargin) + .booleanConf + .createWithDefault(true) + private val FORCED_CHECKSUM_VALIDATION_INTERVAL_DEFAULT = 400 val FORCED_CHECKSUM_VALIDATION_INTERVAL = buildConf("versionChecksum.forcedValidationInterval") @@ -3166,7 +3206,7 @@ trait DeltaSQLConfBase extends DeltaSQLConfUtils { .internal() .doc("Maximum number of files allowed in initial snapshot for V2 streaming.") .intConf - .createWithDefault(50000) + .createWithDefault(100000) } object DeltaSQLConf extends DeltaSQLConfBase diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSink.scala b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSink.scala index d8a559ff32b..10b5037c17e 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSink.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSink.scala @@ -42,7 +42,7 @@ import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} import org.apache.spark.sql.execution.metric.SQLMetrics.createMetric import org.apache.spark.sql.execution.streaming.Sink import org.apache.spark.sql.streaming.OutputMode -import org.apache.spark.sql.types.{ArrayType, DataType, MapType, NullType, StructType} +import org.apache.spark.sql.types.{ArrayType, DataType, MapType, StructType} import org.apache.spark.util.Utils /** @@ -114,7 +114,7 @@ case class DeltaSink( val txn = deltaLog.startTransaction(catalogTable) assert(queryId != null) - if (SchemaUtils.typeExistsRecursively(data.schema)(_.isInstanceOf[NullType])) { + if (SchemaUtils.nullTypeExistsRecursively(data.schema)) { throw DeltaErrors.streamWriteNullTypeException } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingReader.scala b/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingReader.scala index a38739995bb..1685e2bed64 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingReader.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/stats/DataSkippingReader.scala @@ -232,6 +232,27 @@ private[delta] object DataSkippingReader { Option(ExpressionEncoder[java.lang.Long]()), Option(ExpressionEncoder[java.lang.Long]()), Option(ExpressionEncoder[java.lang.Long]())) + + /** + * For timestamps, JSON serialization will truncate to milliseconds. This means + * that we must adjust 1 millisecond upwards for max stats, or we will incorrectly skip + * records that differ only in microsecond precision. (For example, a file containing only + * 01:02:03.456789 will be written with min == max == 01:02:03.456, so we must consider it + * to contain the range from 01:02:03.456 to 01:02:03.457.) + * + * To avoid overflow when the timestamp is near Long.MAX_VALUE, we check if adding 1 + * millisecond would overflow. If so, we saturate to Long.MAX_VALUE to ensure the max stat + * is >= all actual values in the file while avoiding arithmetic overflow. + */ + def getAdjustedTimestamp(col: Column, tsType: DataType): Column = { + val maxTimestampLiteral = Literal(Long.MaxValue, tsType) + val overflowThresholdLiteral = Literal(Long.MaxValue - 1000, tsType) + val adjustedExpr = If( + GreaterThan(col.expr, overflowThresholdLiteral), + maxTimestampLiteral, + TimestampAdd("MILLISECOND", Literal(1L, LongType), col.expr)) + Column(Cast(adjustedExpr, tsType)) + } } /** @@ -1087,27 +1108,9 @@ trait DataSkippingReaderBase .filterNot(_._2.isInstanceOf[StructType]) .map { case (statCol, TimestampType, _) if pathToStatType.head == MAX => - // SC-22824: For timestamps, JSON serialization will truncate to milliseconds. This means - // that we must adjust 1 millisecond upwards for max stats, or we will incorrectly skip - // records that differ only in microsecond precision. (For example, a file containing only - // 01:02:03.456789 will be written with min == max == 01:02:03.456, so we must consider it - // to contain the range from 01:02:03.456 to 01:02:03.457.) - // - // There is a longer term task SC-22825 to fix the serialization problem that caused this. - // But we need the adjustment in any case to correctly read stats written by old versions. - // TimeAdd is removed in Spark 4.1, using TimestampAdd instead - Column(Cast(TimestampAdd( - "MILLISECOND", - new Literal(1L, LongType), - statCol.expr), TimestampType)) + getAdjustedTimestamp(statCol, TimestampType) case (statCol, TimestampNTZType, _) if pathToStatType.head == MAX => - // We also apply the same adjustment of max stats that was applied to Timestamp - // for TimestampNTZ because these 2 types have the same precision in terms of time. - // TimeAdd is removed in Spark 4.1, using TimestampAdd instead - Column(Cast(TimestampAdd( - "MILLISECOND", - new Literal(1L, LongType), - statCol.expr), TimestampNTZType)) + getAdjustedTimestamp(statCol, TimestampNTZType) case (statCol, _, _) => statCol } @@ -1263,7 +1266,10 @@ trait DataSkippingReaderBase partitionFilters: Seq[Expression], keepNumRecords: Boolean): (Seq[AddFile], DataSize) = recordFrameProfile( "Delta", "DataSkippingReader.filterOnPartitions") { - val df = if (keepNumRecords) { + val forceCollectRowCount = + spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_ALWAYS_COLLECT_STATS) + val shouldCollectStats = keepNumRecords || forceCollectRowCount + val df = if (shouldCollectStats) { // use withStats instead of allFiles so the `stats` column is already parsed val filteredFiles = DeltaLog.filterFileList(metadata.partitionSchema, withStats, partitionFilters) @@ -1278,7 +1284,36 @@ trait DataSkippingReaderBase } val files = convertDataFrameToAddFiles(df) val sizeInBytesByPartitionFilters = files.map(_.size).sum - files.toSeq -> DataSize(Some(sizeInBytesByPartitionFilters), None, Some(files.size)) + // Compute row count if we have stats available and forceCollectRowCount is enabled + val (rowCount, logicalRowCount) = if (forceCollectRowCount) { + sumRowCounts(files) + } else { + (None, None) + } + files.toSeq -> DataSize(Some(sizeInBytesByPartitionFilters), rowCount, Some(files.size), + logicalRowCount) + } + + /** + * Sums up the numPhysicalRecords and numLogicalRecords from the given AddFile objects. + * Returns (None, None) if any file is missing physical record stats. + * Returns (Some(physical), None) if any file is missing logical record stats. + */ + private def sumRowCounts(files: Seq[AddFile]): (Option[Long], Option[Long]) = { + var physicalRows = 0L + var logicalRows = 0L + var physicalMissing = false + var logicalMissing = false + files.foreach { file => + physicalMissing = physicalMissing || file.numPhysicalRecords.isEmpty + logicalMissing = logicalMissing || file.numLogicalRecords.isEmpty + physicalRows += file.numPhysicalRecords.getOrElse(0L) + logicalRows += file.numLogicalRecords.getOrElse(0L) + } + ( + if (physicalMissing) None else Some(physicalRows), + if (logicalMissing) None else Some(logicalRows) + ) } /** @@ -1335,13 +1370,24 @@ trait DataSkippingReaderBase if (filters == Seq(TrueLiteral) || filters.isEmpty || schema.isEmpty) { recordDeltaOperation(deltaLog, "delta.skipping.none") { // When there are no filters we can just return allFiles with no extra processing + val forceCollectRowCount = + spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_ALWAYS_COLLECT_STATS) + val shouldCollectStats = keepNumRecords || forceCollectRowCount + lazy val files = getAllFiles(shouldCollectStats) + // Compute row count if forceCollectRowCount is enabled + val (rowCount, logicalRowCount) = if (forceCollectRowCount) { + sumRowCounts(files) + } else { + (None, None) + } val dataSize = DataSize( bytesCompressed = sizeInBytesIfKnown, - rows = None, - files = numOfFilesIfKnown) + rows = rowCount, + files = numOfFilesIfKnown, + logicalRows = logicalRowCount) return DeltaScan( version = version, - files = getAllFiles(keepNumRecords), + files = files, total = dataSize, partition = dataSize, scanned = dataSize)( diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/storage/S3SingleDriverLogStore.scala b/spark/src/main/scala/org/apache/spark/sql/delta/storage/S3SingleDriverLogStore.scala index 528625cc573..cf2cceebafd 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/storage/S3SingleDriverLogStore.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/storage/S3SingleDriverLogStore.scala @@ -198,7 +198,9 @@ class S3SingleDriverLogStore( } catch { // Convert Hadoop's FileAlreadyExistsException to Java's FileAlreadyExistsException case e: org.apache.hadoop.fs.FileAlreadyExistsException => - throw new java.nio.file.FileAlreadyExistsException(e.getMessage) + val converted = new java.nio.file.FileAlreadyExistsException(e.getMessage) + converted.initCause(e) + throw converted } finally { releasePathLock(lockedPath) } diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2TableManager.scala b/spark/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2TableManager.scala new file mode 100644 index 00000000000..73354dfb7d0 --- /dev/null +++ b/spark/src/main/scala/org/apache/spark/sql/delta/v2/interop/DeltaV2TableManager.scala @@ -0,0 +1,67 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.v2.interop + +import org.apache.spark.sql.delta.{DeltaLog, Snapshot} +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.Path + +import org.apache.spark.util.Clock + +/** + * Interface for accessing and managing a Delta table. Mirrors DeltaLog's role as the entry + * point for all Delta table operations. + * + * Two implementations: + * - [[DeltaLog]] (existing): reads the Delta protocol via log replay + * - KernelTableManager (future): reads the Delta protocol via Kernel + * + * Commands, OptimisticTransaction, and other business logic code against this interface + * so they work with either backend. + */ +trait DeltaV2TableManager { + + /** The path of the Delta log directory. */ + def logPath: Path + + /** The path of the data directory of the Delta table. */ + def dataPath: Path + + /** Clock used for timestamping operations. */ + def clock: Clock + + /** Hadoop configuration options for this table. */ + def options: Map[String, String] + + /** Create a new Hadoop Configuration for this table. */ + def newDeltaHadoopConf(): Configuration + + /** Get the current snapshot of the table, updating if necessary. */ + def snapshot: Snapshot + + /** Get the latest snapshot without checking for updates. */ + def unsafeVolatileSnapshot: Snapshot + + /** Whether the table exists (version >= 0). */ + def tableExists: Boolean + + /** A unique identifier for this table. */ + def tableId: String + + /** Create the log directories if they do not exist. */ + def createLogDirectoriesIfNotExists(): Unit +} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaAllFilesInCrcSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaAllFilesInCrcSuite.scala index 29594434e4f..b0fb2eb0394 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaAllFilesInCrcSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaAllFilesInCrcSuite.scala @@ -171,6 +171,7 @@ class DeltaAllFilesInCrcSuite try { val query = s"DELETE from delta.`${tempDir.getAbsolutePath}` WHERE id >= 0" val (observer, future) = runQueryWithObserver(name = "A", executor, query) + observer.phases.analysisPhase.entryBarrier.unblock() observer.phases.initialPhase.entryBarrier.unblock() observer.phases.preparePhase.entryBarrier.unblock() // Make sure that delete query has run the actual computation and has reached diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaCDCSQLSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaCDCSQLSuite.scala index 16d0fe00682..58a2947258c 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaCDCSQLSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaCDCSQLSuite.scala @@ -294,6 +294,36 @@ class DeltaCDCSQLSuite extends DeltaCDCSuiteBase with DeltaColumnMappingTestUtil } } + test("negative case - table_changes in correlated subquery with OuterReference") { + // When table_changes() is used inside a correlated subquery with an expression that + // wraps an OuterReference (e.g. `o.version + 0`), the top-level node is not Unevaluable + // (Add is evaluable) but its child OuterReference is. The old isInstanceOf[Unevaluable] + // check on the top-level expression misses this case and .eval() throws INTERNAL_ERROR. + // We instead catch that SparkException and re-throw as DELTA_CDC_NON_CONSTANT_ARGUMENT. + val tbl = "tbl" + val otherTbl = "other_tbl" + withTable(tbl, otherTbl) { + spark.range(10).write.format("delta").saveAsTable(tbl) + spark.range(5).toDF("version").write.format("delta").saveAsTable(otherTbl) + + val q = s""" + SELECT * FROM $otherTbl o WHERE EXISTS ( + SELECT 1 FROM table_changes('$tbl', o.version + 0) + ) + """ + checkErrorMatchPVals( + intercept[AnalysisException] { sql(q) }, + "DELTA_CDC_NON_CONSTANT_ARGUMENT", + parameters = Map( + "argumentName" -> "`starting`", + "pos" -> "2", + "functionName" -> "`table_changes`", + "sqlExpr" -> ".*" + ) + ) + } + } + test("resolve expression for timestamp function") { val tbl = "tbl" withDefaultTimeZone(UTC) { diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaColumnMappingSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaColumnMappingSuite.scala index 52eb9c053dc..af9a9b346fd 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaColumnMappingSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaColumnMappingSuite.scala @@ -1413,9 +1413,14 @@ class DeltaColumnMappingSuite extends QueryTest // case 1: currentSchema compared with itself var currentMetadata = getMetadata() var newMetadata = getMetadata() + def isBothCMEnabled: Boolean = + newMetadata.columnMappingMode != NoMapping && + currentMetadata.columnMappingMode != NoMapping assert( - !isDropColumnOperation(newMetadata, currentMetadata) && - !isRenameColumnOperation(newMetadata, currentMetadata) + !isDropColumnOperation( + newMetadata.schema, currentMetadata.schema, isBothCMEnabled) && + !isRenameColumnOperation( + newMetadata.schema, currentMetadata.schema, isBothCMEnabled) ) // case 2: add a top-level column @@ -1423,8 +1428,10 @@ class DeltaColumnMappingSuite extends QueryTest currentMetadata = newMetadata newMetadata = getMetadata() assert( - !isDropColumnOperation(newMetadata, currentMetadata) && - !isRenameColumnOperation(newMetadata, currentMetadata) + !isDropColumnOperation( + newMetadata.schema, currentMetadata.schema, isBothCMEnabled) && + !isRenameColumnOperation( + newMetadata.schema, currentMetadata.schema, isBothCMEnabled) ) // case 3: add a nested column @@ -1432,8 +1439,10 @@ class DeltaColumnMappingSuite extends QueryTest currentMetadata = newMetadata newMetadata = getMetadata() assert( - !isDropColumnOperation(newMetadata, currentMetadata) && - !isRenameColumnOperation(newMetadata, currentMetadata) + !isDropColumnOperation( + newMetadata.schema, currentMetadata.schema, isBothCMEnabled) && + !isRenameColumnOperation( + newMetadata.schema, currentMetadata.schema, isBothCMEnabled) ) // case 4: drop a top-level column @@ -1441,8 +1450,10 @@ class DeltaColumnMappingSuite extends QueryTest currentMetadata = newMetadata newMetadata = getMetadata() assert( - isDropColumnOperation(newMetadata, currentMetadata) && - !isRenameColumnOperation(newMetadata, currentMetadata) + isDropColumnOperation( + newMetadata.schema, currentMetadata.schema, isBothCMEnabled) && + !isRenameColumnOperation( + newMetadata.schema, currentMetadata.schema, isBothCMEnabled) ) // case 5: drop a nested column @@ -1450,8 +1461,10 @@ class DeltaColumnMappingSuite extends QueryTest currentMetadata = newMetadata newMetadata = getMetadata() assert( - isDropColumnOperation(newMetadata, currentMetadata) && - !isRenameColumnOperation(newMetadata, currentMetadata) + isDropColumnOperation( + newMetadata.schema, currentMetadata.schema, isBothCMEnabled) && + !isRenameColumnOperation( + newMetadata.schema, currentMetadata.schema, isBothCMEnabled) ) // case 6: rename a top-level column @@ -1459,8 +1472,10 @@ class DeltaColumnMappingSuite extends QueryTest currentMetadata = newMetadata newMetadata = getMetadata() assert( - !isDropColumnOperation(newMetadata, currentMetadata) && - isRenameColumnOperation(newMetadata, currentMetadata) + !isDropColumnOperation( + newMetadata.schema, currentMetadata.schema, isBothCMEnabled) && + isRenameColumnOperation( + newMetadata.schema, currentMetadata.schema, isBothCMEnabled) ) // case 7: rename a nested column @@ -1468,8 +1483,10 @@ class DeltaColumnMappingSuite extends QueryTest currentMetadata = newMetadata newMetadata = getMetadata() assert( - !isDropColumnOperation(newMetadata, currentMetadata) && - isRenameColumnOperation(newMetadata, currentMetadata) + !isDropColumnOperation( + newMetadata.schema, currentMetadata.schema, isBothCMEnabled) && + isRenameColumnOperation( + newMetadata.schema, currentMetadata.schema, isBothCMEnabled) ) } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaCreateTableLikeSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaCreateTableLikeSuite.scala index ec1abc4a5ef..396822e2068 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaCreateTableLikeSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaCreateTableLikeSuite.scala @@ -17,12 +17,15 @@ package org.apache.spark.sql.delta import java.io.File -import java.net.URI import java.util.UUID import org.apache.spark.sql.delta.catalog.DeltaCatalog -import org.apache.spark.sql.delta.commands.{CreateDeltaTableCommand, TableCreationModes} -import org.apache.spark.sql.delta.test.DeltaSQLCommandTest +import org.apache.spark.sql.delta.commands.{ + CreateDeltaTableCommand, + CreateDeltaTableLike, + TableCreationModes +} +import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils} import org.scalatest.exceptions.TestFailedException import org.apache.spark.sql.QueryTest @@ -35,7 +38,8 @@ import org.apache.spark.sql.types.StructType class DeltaCreateTableLikeSuite extends QueryTest with SharedSparkSession - with DeltaSQLCommandTest { + with DeltaSQLCommandTest + with DeltaSQLTestUtils { def checkTableEmpty(tblName: String): Boolean = { val numRows = spark.sql(s"SELECT * FROM $tblName") @@ -289,7 +293,7 @@ class DeltaCreateTableLikeSuite extends QueryTest withTable("t") { def getCatalogTable: CatalogTable = { val storage = CatalogStorageFormat.empty.copy( - locationUri = Some(new URI(s"$dir/${UUID.randomUUID().toString}"))) + locationUri = Some(dir.toPath.resolve(UUID.randomUUID().toString).toUri)) val catalogTableTarget = CatalogTable( identifier = TableIdentifier("t"), tableType = CatalogTableType.MANAGED, @@ -320,6 +324,102 @@ class DeltaCreateTableLikeSuite extends QueryTest } } + test("catalog-managed CREATE OR REPLACE creates missing tables") { + withTempDir { dir => + withTable("t") { + val storage = CatalogStorageFormat.empty.copy( + locationUri = Some(dir.toPath.resolve(UUID.randomUUID().toString).toUri)) + val catalogTableTarget = CatalogTable( + identifier = TableIdentifier("t"), + tableType = CatalogTableType.MANAGED, + storage = storage, + provider = Some("delta"), + schema = new StructType().add("id", "long")) + val command = CreateDeltaTableCommand( + new DeltaCatalog().verifyTableAndSolidify( + tableDesc = catalogTableTarget, + query = None, + maybeClusterBySpec = None), + existingTableOpt = None, + mode = SaveMode.ErrorIfExists, + query = None, + operation = TableCreationModes.CreateOrReplace, + allowCatalogManaged = true, + createTableFunc = None) + + command.run(spark) + assert(spark.sessionState.catalog.tableExists(TableIdentifier("t"))) + } + } + } + + test("catalog-managed CREATE OR REPLACE skips catalog create callback " + + "when metadata is unchanged") { + withCatalogManagedTable(createTable = false) { tableName => + spark.sql( + s"""CREATE TABLE $tableName (id LONG) USING DELTA + |TBLPROPERTIES ('delta.feature.catalogManaged' = 'supported') + |""".stripMargin) + + val existingTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(tableName)) + val snapshot = DeltaLog.forTable(spark, existingTable).update() + var createCallbackCalls = 0 + + val command = new CreateDeltaTableLike { + override val table: CatalogTable = existingTable + override val existingTableOpt: Option[CatalogTable] = Some(existingTable) + override val operation: TableCreationModes.CreationMode = + TableCreationModes.CreateOrReplace + override val mode: SaveMode = SaveMode.ErrorIfExists + override val allowCatalogManaged: Boolean = true + + def runUpdateCatalog(): Unit = { + updateCatalog( + spark, + table, + snapshot, + query = None, + didNotChangeMetadata = true, + createTableFunc = Some((_: CatalogTable) => { + createCallbackCalls += 1 + })) + } + } + + command.runUpdateCatalog() + assert(createCallbackCalls === 0) + } + } + + test("catalog-managed CREATE OR REPLACE rejects query-derived nullable schema") { + withCatalogManagedTable(createTable = false) { tableName => + withTable("source") { + spark.sql( + s"""CREATE TABLE $tableName (id LONG NOT NULL) USING DELTA + |TBLPROPERTIES ('delta.feature.catalogManaged' = 'supported') + |""".stripMargin) + spark.sql(s"INSERT INTO $tableName VALUES (1)") + spark.sql("CREATE TABLE source (id LONG) USING DELTA") + spark.sql("INSERT INTO source VALUES (2)") + + val existingTable = + spark.sessionState.catalog.getTableMetadata(TableIdentifier(tableName)) + val versionBefore = DeltaLog.forTable(spark, existingTable).update().version + val query = spark.sql("SELECT id FROM source").logicalPlan + val err = intercept[AssertionError] { + new DeltaCatalog().verifyTableAndSolidify( + tableDesc = existingTable.copy(schema = query.schema.asNullable), + query = Some(query), + maybeClusterBySpec = None) + } + + assert(err.getMessage.contains("Can't specify table schema in CTAS.")) + assert(DeltaLog.forTable(spark, existingTable).update().version === versionBefore) + checkAnswer(spark.sql(s"SELECT * FROM $tableName"), Seq(org.apache.spark.sql.Row(1L))) + } + } + } + test("CREATE TABLE LIKE where sourceTable is a json table") { val srcTbl = "srcTbl" val targetTbl = "targetTbl" @@ -370,7 +470,7 @@ class DeltaCreateTableLikeSuite extends QueryTest test("CREATE TABLE LIKE where target table is a named external table") { val srcTbl = "srcTbl" val targetTbl = "targetTbl" - withTempDir { dir => + withTempDir(prefix = "sparkdirprefix") { dir => withTable(srcTbl) { createTable(srcTbl) spark.sql(s"CREATE TABLE $targetTbl LIKE $srcTbl LOCATION '${dir.toURI.toString}'") @@ -385,7 +485,7 @@ class DeltaCreateTableLikeSuite extends QueryTest withTable(srcTbl) { createTable(srcTbl) spark.sql(s"CREATE TABLE delta.`${dir.toURI.toString}` LIKE $srcTbl") - checkTableCopyDelta(srcTbl, dir.toString, checkTargetTableByPath = true + checkTableCopyDelta(srcTbl, dir.toURI.toString, checkTargetTableByPath = true ) } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoSchemaEvolutionSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoSchemaEvolutionSuite.scala index aabdc600faf..abfd74cb3ac 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoSchemaEvolutionSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoSchemaEvolutionSuite.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql.delta import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.SparkThrowable import org.apache.spark.sql.SaveMode import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ @@ -97,9 +98,8 @@ class DeltaInsertIntoSchemaEvolutionSuite extends DeltaInsertIntoTest { ) - // Adding new top-level columns with schema evolution is allowed for all inserts except SQL - // inserts by name, but dataframe inserts by name don't support implicit casting and will fail - // due to the type mismatch. + // Adding new top-level columns with schema evolution is allowed for all inserts, but dataframe + // inserts by name don't support implicit casting and will fail due to the type mismatch. testInserts(s"insert with extra top-level column and implicit cast," + s"schemaEvolution=$schemaEvolution")( initialData = TestData("a int, b int", Seq("""{ "a": 1, "b": 2 }""")), @@ -140,22 +140,32 @@ class DeltaInsertIntoSchemaEvolutionSuite extends DeltaInsertIntoTest { withSchemaEvolution = schemaEvolution ) + // SQL inserts by name fail with a different error in the analysis when there's an extra column + // and schema evolution is disabled. testInserts(s"insert with extra top-level column and implicit cast," + s"schemaEvolution=$schemaEvolution")( initialData = TestData("a int, b int", Seq("""{ "a": 1, "b": 2 }""")), partitionBy = Seq("a"), overwriteWhere = "a" -> 1, insertData = TestData("a int, b long, c int", Seq("""{ "a": 1, "b": 4, "c": 5 }""")), - expectedResult = ExpectedResult.Failure(ex => { - checkError( - ex, - "INSERT_COLUMN_ARITY_MISMATCH.TOO_MANY_DATA_COLUMNS", - parameters = Map( - "tableName" -> s"`$catalogName`.`default`.`target`", - "tableColumns" -> "`a`, `b`", - "dataColumns" -> "`a`, `b`, `c`" - )) - }), + expectedResult = if (schemaEvolution) { + ExpectedResult.Success( + expected = new StructType() + .add("a", IntegerType) + .add("b", IntegerType) + .add("c", IntegerType)) + } else { + ExpectedResult.Failure(ex => { + checkError( + ex, + "INSERT_COLUMN_ARITY_MISMATCH.TOO_MANY_DATA_COLUMNS", + parameters = Map( + "tableName" -> s"`$catalogName`.`default`.`target`", + "tableColumns" -> "`a`, `b`", + "dataColumns" -> "`a`, `b`, `c`" + )) + }) + }, includeInserts = insertsSQL.intersect(insertsByName) -- Set( // It's not possible to specify a column that doesn't exist in the target using SQL with an // explicit column list. @@ -273,4 +283,28 @@ class DeltaInsertIntoSchemaEvolutionSuite extends DeltaInsertIntoTest { withSchemaEvolution = schemaEvolution ) } + + // When DELTA_INSERT_BY_NAME_SCHEMA_EVOLUTION_ENABLED is disabled, SQL INSERT BY NAME with extra + // top-level columns should fail even when schema evolution is enabled. + test("insert by name with extra top-level column and implicit cast fails " + + "when byNameSchemaEvolution is disabled") { + withTable("target") { + sql("CREATE TABLE target (a INT, b INT) USING DELTA") + withSQLConf( + DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE.key -> "true", + DeltaSQLConf.DELTA_INSERT_BY_NAME_SCHEMA_EVOLUTION_ENABLED.key -> "false") { + val ex = intercept[SparkThrowable] { + sql("INSERT INTO target BY NAME SELECT 1 AS a, 2L AS b, 3 AS c") + } + checkError( + ex, + "INSERT_COLUMN_ARITY_MISMATCH.TOO_MANY_DATA_COLUMNS", + parameters = Map( + "tableName" -> s"`$catalogName`.`default`.`target`", + "tableColumns" -> "`a`, `b`", + "dataColumns" -> "`a`, `b`, `c`" + )) + } + } + } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoTableSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoTableSuite.scala index 2ddd9397305..c9f468c2adf 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoTableSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaInsertIntoTableSuite.scala @@ -168,7 +168,8 @@ class DeltaInsertIntoSQLSuite // true (due to case sensitivity) so that we call resolveQueryColumnsByName and hit the right // code path. - // when the number of columns does not match, throw an arity mismatch error. + // when the number of columns does not match and schema evolution is disabled, throw + // an arity mismatch error. testInsertByNameError( targetSchema = "(A int)", expectedErrorClass = "INSERT_COLUMN_ARITY_MISMATCH.TOO_MANY_DATA_COLUMNS") diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaOptionSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaOptionSuite.scala index 8ea7a1b36f9..0e629582a99 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaOptionSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaOptionSuite.scala @@ -19,7 +19,7 @@ package org.apache.spark.sql.delta import java.util.Locale // scalastyle:off import.ordering.noEmptyLine -import org.apache.spark.sql.delta.DeltaOptions.{OVERWRITE_SCHEMA_OPTION, PARTITION_OVERWRITE_MODE_OPTION} +import org.apache.spark.sql.delta.DeltaOptions.{OVERWRITE_SCHEMA_OPTION, PARTITION_OVERWRITE_MODE_OPTION, USE_NULL_INTOLERANT_EQUALITY_WITH_DPO} import org.apache.spark.sql.delta.actions.{Action, FileAction} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.DeltaSQLCommandTest @@ -312,6 +312,30 @@ class DeltaOptionSuite extends QueryTest } } + test("useNullIntolerantEqualityWithDPO should never be specified when " + + "Dynamic Partition Overwrite is not enabled") { + withTable("temp") { + checkError( + exception = intercept[DeltaIllegalArgumentException] { + Seq(1, 2, 3).toDF + .withColumn("part", $"value" % 2) + .write + .mode("overwrite") + .format("delta") + .partitionBy("part") + .option(USE_NULL_INTOLERANT_EQUALITY_WITH_DPO, "true") + .saveAsTable("temp") + }, + condition = "DELTA_ILLEGAL_OPTION", + sqlState = Some("42616"), + parameters = Map( + "name" -> USE_NULL_INTOLERANT_EQUALITY_WITH_DPO, + "input" -> "true", + "explain" -> "This option should be specified only in Dynamic Partition Overwrite mode.") + ) + } + } + test("overwriteSchema=true should be invalid with partitionOverwriteMode=dynamic, " + "saveAsTable") { withTable("temp") { diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaRetentionSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaRetentionSuite.scala index 2552ffabf52..506e882fbf0 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaRetentionSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaRetentionSuite.scala @@ -130,10 +130,9 @@ class DeltaRetentionSuite extends QueryTest deltaFile.setLastModified(clock.getTimeMillis() + i * 10000) val crcFile = new File(FileNames.checksumFile(log.logPath, version).toUri) crcFile.setLastModified(clock.getTimeMillis() + i * 10000) - val chk = new File(FileNames.checkpointFileSingular(log.logPath, version).toUri) - if (chk.exists()) { - chk.setLastModified(clock.getTimeMillis() + i * 10000) - } + getCheckpointFiles(logPath) + .filter(f => FileNames.checkpointVersion(new Path(f.getCanonicalPath)) == version) + .foreach(_.setLastModified(clock.getTimeMillis() + i * 10000)) } // delete some files in the middle diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSinkSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSinkSuite.scala index 766e6ed9aec..9a78364b0c9 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSinkSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSinkSuite.scala @@ -655,6 +655,31 @@ class DeltaSinkSuite "deltaLog should not be initialized after constructor") } } + + test("DeltaSink rejects DataFrame with UDT containing NullType") { + failAfter(streamingTimeout) { + withTempDirs { (outputDir, checkpointDir) => + val inputData = MemoryStream[Int] + val ds = inputData.toDS() + val dsWriter = + ds.map(i => (i, new NullData())) + .toDF("id", "value") + .writeStream + .option("checkpointLocation", checkpointDir.getCanonicalPath) + .format("delta") + + val wrapperException = intercept[StreamingQueryException] { + val q = dsWriter.start(outputDir.getCanonicalPath) + inputData.addData(42) + q.processAllAvailable() + } + assert(wrapperException.cause.isInstanceOf[AnalysisException]) + checkError( + wrapperException.cause.asInstanceOf[AnalysisException], + "DELTA_NULL_SCHEMA_IN_STREAMING_WRITE") + } + } + } } // Batch sizes 1, 2, and 100 exercise different backfill behaviors in the commit coordinator. diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceDeletionVectorsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceDeletionVectorsSuite.scala index daaca6b70c1..42b7e43d079 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceDeletionVectorsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceDeletionVectorsSuite.scala @@ -31,9 +31,17 @@ import org.apache.spark.sql.streaming.util.StreamManualClock trait DeltaSourceDeletionVectorTests extends StreamTest with DeletionVectorsTestUtils { + self: DeltaSourceConnectorTrait => import testImplicits._ + /** + * Executes a DML SQL statement (DELETE, INSERT, etc.). + * Overridable so that V2 suites can route DML through the V1 connector, + * since SparkTable (V2) is read-only and does not support writes. + */ + protected def executeDml(sqlText: String): Unit = sql(sqlText) + test("allow to delete files before starting a streaming query") { withTempDir { inputDir => val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) @@ -41,7 +49,7 @@ trait DeltaSourceDeletionVectorTests extends StreamTest val v = Seq(i.toString).toDF v.write.mode("append").format("delta").save(deltaLog.dataPath.toString) } - sql(s"DELETE FROM delta.`$inputDir`") + executeDml(s"DELETE FROM delta.`$inputDir`") (5 until 10).foreach { i => val v = Seq(i.toString).toDF v.write.mode("append").format("delta").save(deltaLog.dataPath.toString) @@ -49,9 +57,7 @@ trait DeltaSourceDeletionVectorTests extends StreamTest deltaLog.checkpoint() assert(deltaLog.readLastCheckpointFile().nonEmpty, "this test requires a checkpoint") - val df = spark.readStream - .format("delta") - .load(inputDir.getCanonicalPath) + val df = loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) testStream(df)( AssertOnQuery { q => @@ -69,16 +75,14 @@ trait DeltaSourceDeletionVectorTests extends StreamTest val v = Seq(i.toString).toDF v.write.mode("append").format("delta").save(deltaLog.dataPath.toString) } - sql(s"DELETE FROM delta.`$inputDir`") + executeDml(s"DELETE FROM delta.`$inputDir`") (5 until 7).foreach { i => val v = Seq(i.toString).toDF v.write.mode("append").format("delta").save(deltaLog.dataPath.toString) } assert(deltaLog.readLastCheckpointFile().isEmpty, "this test requires no checkpoint") - val df = spark.readStream - .format("delta") - .load(inputDir.getCanonicalPath) + val df = loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) testStream(df)( AssertOnQuery { q => @@ -115,7 +119,7 @@ trait DeltaSourceDeletionVectorTests extends StreamTest Seq(i, i + 1).toDF().coalesce(1).write.format("delta").mode("append").save(inputDir) } - val df = spark.readStream.format("delta").options(sourceOptions.toMap).load(inputDir) + val df = loadStreamWithOptions(inputDir, sourceOptions.toMap) val expectDVs = commandShouldProduceDVs.getOrElse( sqlCommand.toUpperCase().startsWith("DELETE")) @@ -126,7 +130,7 @@ trait DeltaSourceDeletionVectorTests extends StreamTest }, CheckAnswer((0 until 10): _*), AssertOnQuery { q => - sql(sqlCommand) + executeDml(sqlCommand) deletionVectorsPresentIfExpected(inputDir, expectDVs) }) @@ -148,7 +152,7 @@ trait DeltaSourceDeletionVectorTests extends StreamTest } val log = DeltaLog.forTable(spark, inputDir) val commitVersionBeforeDML = log.update().version - val df = spark.readStream.format("delta").options(sourceOptions.toMap).load(inputDir) + val df = loadStreamWithOptions(inputDir, sourceOptions.toMap) def expectDVsInCommand(shouldProduceDVs: Option[Boolean], command: String): Boolean = { shouldProduceDVs.getOrElse(command.toUpperCase().startsWith("DELETE")) } @@ -177,11 +181,11 @@ trait DeltaSourceDeletionVectorTests extends StreamTest true }, AssertOnQuery { q => - sql(sqlCommand1) + executeDml(sqlCommand1) deletionVectorsPresentIfExpected(inputDir, expectDVsInCommand1) }, AssertOnQuery { q => - sql(sqlCommand2) + executeDml(sqlCommand2) deletionVectorsPresentIfExpected(inputDir, expectDVsInCommand2) }, AssertOnQuery { q => @@ -416,21 +420,19 @@ trait DeltaSourceDeletionVectorTests extends StreamTest (0 until 10).toDF("value").coalesce(1).write.format("delta").save(path) // V1: Delete row 0 - sql(s"DELETE FROM delta.`$path` WHERE value = 0") + executeDml(s"DELETE FROM delta.`$path` WHERE value = 0") // V2: Delete row 1 - sql(s"DELETE FROM delta.`$path` WHERE value = 1") + executeDml(s"DELETE FROM delta.`$path` WHERE value = 1") // V3: Delete row 2 - sql(s"DELETE FROM delta.`$path` WHERE value = 2") + executeDml(s"DELETE FROM delta.`$path` WHERE value = 2") // Verify DVs are present assert(getFilesWithDeletionVectors(deltaLog).nonEmpty, "This test requires deletion vectors to be present") - val df = spark.readStream - .format("delta") - .load(path) + val df = loadStreamWithOptions(path, Map.empty) testStream(df)( // Process the initial snapshot @@ -457,10 +459,7 @@ trait DeltaSourceDeletionVectorTests extends StreamTest // V0: 10 rows in a single file (0 until 10).toDF("value").coalesce(1).write.format("delta").save(path) - val df = spark.readStream - .format("delta") - .options(sourceOptions.toMap) - .load(path) + val df = loadStreamWithOptions(path, sourceOptions.toMap) testStream(df)( AssertOnQuery { q => @@ -470,12 +469,12 @@ trait DeltaSourceDeletionVectorTests extends StreamTest CheckAnswer((0 until 10): _*), AssertOnQuery { q => // V1: Delete row 0 - creates first DV (version 1) - sql(s"DELETE FROM delta.`$path` WHERE value = 0") + executeDml(s"DELETE FROM delta.`$path` WHERE value = 0") true }, AssertOnQuery { q => // V2: Delete row 1 - updates DV (version 2). DV is cumulative: {0, 1} - sql(s"DELETE FROM delta.`$path` WHERE value = 1") + executeDml(s"DELETE FROM delta.`$path` WHERE value = 1") true }, AssertOnQuery { q => diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSuite.scala index 1abb516c5d6..cf21375299f 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSourceSuite.scala @@ -47,7 +47,7 @@ import org.apache.spark.sql.execution.streaming._ import org.apache.spark.sql.functions.when import org.apache.spark.sql.streaming.{OutputMode, StreamingQuery, StreamingQueryException, Trigger} import org.apache.spark.sql.streaming.util.StreamManualClock -import org.apache.spark.sql.types.{IntegerType, NullType, StringType, StructField, StructType} +import org.apache.spark.sql.types.{IntegerType, LongType, NullType, StringType, StructField, StructType} import org.apache.spark.unsafe.types.UTF8String import org.apache.spark.util.{ManualClock, Utils} @@ -1820,7 +1820,7 @@ class DeltaSourceSuite extends DeltaSourceSuiteBase } } - testQuietly("SC-46515: deltaSourceIgnoreChangesError contains removeFile, version, tablePath") { + testQuietly("deltaSourceIgnoreChangesError contains removeFile, version, tablePath") { withTempDirs { (inputDir, outputDir, checkpointDir) => Seq(1, 2, 3).toDF("x").write.format("delta").save(inputDir.toString) val df = loadStreamWithOptions(inputDir.toString, Map.empty) @@ -1862,7 +1862,7 @@ class DeltaSourceSuite extends DeltaSourceSuiteBase } } - testQuietly("SC-46515: deltaSourceIgnoreDeleteError contains removeFile, version, tablePath") { + testQuietly("deltaSourceIgnoreDeleteError contains removeFile, version, tablePath") { withTempDirs { (inputDir, outputDir, checkpointDir) => Seq(1, 2, 3).toDF("x").write.format("delta").save(inputDir.toString) val df = loadStreamWithOptions(inputDir.toString, Map.empty) @@ -1899,6 +1899,177 @@ class DeltaSourceSuite extends DeltaSourceSuiteBase } } + test("streaming with ignoreDeletes = true skips delete-only commits") { + withTempDirs { (inputDir, outputDir, checkpointDir) => + // Write initial data + Seq(1, 2, 3).toDF("x").write.format("delta").save(inputDir.toString) + + val df = loadStreamWithOptions( + inputDir.toString, Map(DeltaOptions.IGNORE_DELETES_OPTION -> "true", + "startingVersion" -> "0")) + + val q = df.writeStream + .format("delta") + .option("checkpointLocation", checkpointDir.toString) + .start(outputDir.toString) + try { + q.processAllAvailable() + checkAnswer( + spark.read.format("delta").load(outputDir.toString), + Seq(1, 2, 3).map(Row(_))) + + // Delete all rows: produces only RemoveFile actions + io.delta.tables.DeltaTable.forPath(spark, inputDir.getAbsolutePath).delete() + + // Append new data after the delete + Seq(4, 5).toDF("x").write.format("delta").mode("append").save(inputDir.toString) + + q.processAllAvailable() + + // The delete commit should be silently skipped; only inserts are processed + checkAnswer( + spark.read.format("delta").load(outputDir.toString), + Seq(1, 2, 3, 4, 5).map(Row(_))) + } finally { + q.stop() + } + } + } + + testQuietly("streaming with ignoreDeletes = true still fails on change commits") { + withTempDirs { (inputDir, outputDir, checkpointDir) => + Seq(1, 2, 3).toDF("x").write.format("delta").save(inputDir.toString) + + val df = loadStreamWithOptions( + inputDir.toString, Map(DeltaOptions.IGNORE_DELETES_OPTION -> "true")) + df.writeStream + .format("delta") + .option("checkpointLocation", checkpointDir.toString) + .start(outputDir.toString) + .processAllAvailable() + + // Overwrite produces both AddFile and RemoveFile actions (a change commit) + Seq(4, 5, 6).toDF("x") + .write + .mode("overwrite") + .format("delta") + .save(inputDir.toString) + + val e = intercept[StreamingQueryException] { + val q = df.writeStream + .format("delta") + .option("checkpointLocation", checkpointDir.toString) + .start(outputDir.toString) + + try { + q.processAllAvailable() + } finally { + q.stop() + } + } + + assert(e.getCause.isInstanceOf[UnsupportedOperationException]) + assert(e.getCause.getMessage.contains( + "This is currently not supported. If this is going to happen regularly and you are okay" + + " to skip changes, set the option 'skipChangeCommits' to 'true'." + )) + } + } + + test("streaming with skipChangeCommits = true skips both delete and change commits") { + withTempDirs { (inputDir, outputDir, checkpointDir) => + Seq(1, 2, 3).toDF("x").write.format("delta").save(inputDir.toString) + + val df = loadStreamWithOptions( + inputDir.toString, Map(DeltaOptions.SKIP_CHANGE_COMMITS_OPTION -> "true")) + + val q = df.writeStream + .format("delta") + .option("checkpointLocation", checkpointDir.toString) + .start(outputDir.toString) + try { + q.processAllAvailable() + checkAnswer( + spark.read.format("delta").load(outputDir.toString), + Seq(1, 2, 3).map(Row(_))) + + // Delete all rows: produces only RemoveFile actions (delete-only commit) + io.delta.tables.DeltaTable.forPath(spark, inputDir.getAbsolutePath).delete() + + Seq(4, 5).toDF("x").write.format("delta").mode("append").save(inputDir.toString) + + // Overwrite produces both AddFile and RemoveFile actions (change commit) + Seq(6, 7, 8).toDF("x") + .write + .mode("overwrite") + .format("delta") + .save(inputDir.toString) + + Seq(9, 10).toDF("x").write.format("delta").mode("append").save(inputDir.toString) + + q.processAllAvailable() + + // Both the delete and overwrite commits are silently skipped; only inserts are processed + checkAnswer( + spark.read.format("delta").load(outputDir.toString), + Seq(1, 2, 3, 4, 5, 9, 10).map(Row(_))) + } finally { + q.stop() + } + } + } + + test("streaming with ignoreChanges = true allows both delete and change commits") { + withTempDirs { (inputDir, outputDir, checkpointDir) => + Seq(1, 2, 3).toDF("x").write.format("delta").save(inputDir.toString) + + val df = loadStreamWithOptions( + inputDir.toString, Map(DeltaOptions.IGNORE_CHANGES_OPTION -> "true")) + + val q = df.writeStream + .format("delta") + .option("checkpointLocation", checkpointDir.toString) + .start(outputDir.toString) + try { + q.processAllAvailable() + checkAnswer( + spark.read.format("delta").load(outputDir.toString), + Seq(1, 2, 3).map(Row(_))) + + // Delete all rows: delete-only commit, allowed by ignoreChanges + io.delta.tables.DeltaTable.forPath(spark, inputDir.getAbsolutePath).delete() + + // The delete commit is skipped (no AddFiles) + Seq(4, 5).toDF("x").write.format("delta").mode("append").save(inputDir.toString) + q.processAllAvailable() + checkAnswer( + spark.read.format("delta").load(outputDir.toString), + Seq(1, 2, 3, 4, 5).map(Row(_))) + + // Overwrite produces both AddFile and RemoveFile actions (change commit). + // Unlike skipChangeCommits which skips the commit, ignoreChanges lets the + // AddFiles through (potentially duplicating data). + Seq(6, 7, 8).toDF("x") + .write + .mode("overwrite") + .format("delta") + .save(inputDir.toString) + + Seq(9, 10).toDF("x").write.format("delta").mode("append").save(inputDir.toString) + + q.processAllAvailable() + + // The overwrite commit's AddFiles ARE emitted (unlike skipChangeCommits). The + // result includes the overwrite's new rows. + checkAnswer( + spark.read.format("delta").load(outputDir.toString), + Seq(1, 2, 3, 4, 5, 6, 7, 8, 9, 10).map(Row(_))) + } finally { + q.stop() + } + } + } + test("fail on data loss - starting from missing files") { withTempDirs { (srcData, targetData, chkLocation) => def addData(): Unit = { @@ -2188,6 +2359,7 @@ class DeltaSourceSuite extends DeltaSourceSuiteBase .writeStream .option("checkpointLocation", checkpointDir.getCanonicalPath) .option("mergeSchema", "true") + // use delta sink because we need to check the result .format("delta") .start(outputDir.getCanonicalPath) } @@ -2275,6 +2447,460 @@ class DeltaSourceSuite extends DeltaSourceSuiteBase } } + test("relax nullability: restarting with new DataFrame should recover") { + withTempDirs { (inputDir, outputDir, checkpointDir) => + sql(s"CREATE TABLE delta.`${inputDir.getCanonicalPath}` " + + "(id STRING NOT NULL) USING DELTA") + val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) + (0 until 2).foreach { i => + Seq(i.toString).toDF("id") + .write.mode("append").format("delta").save(deltaLog.dataPath.toString) + } + + def startQuery(): StreamingQuery = { + loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) + .writeStream + .option("checkpointLocation", checkpointDir.getCanonicalPath) + .option("mergeSchema", "true") + // use delta sink because we need to check the result + .format("delta") + .start(outputDir.getCanonicalPath) + } + + var q = startQuery() + try { + q.processAllAvailable() + + // Clear delta log cache + DeltaLog.clearCache() + // Relax nullability from "id STRING NOT NULL" to "id STRING" using the non-cached + // `DeltaLog` to mimic the case that the schema change happens on a different cluster + withMetadata(deltaLog, StructType.fromDDL("id STRING")) + // Insert a null row + Seq(Option.empty[String]).toDF("id") + .write.mode("append").format("delta").save(deltaLog.dataPath.toString) + + // The streaming query should fail when detecting a schema change + val e = intercept[StreamingQueryException] { + q.processAllAvailable() + } + assert(e.getMessage.contains("Detected schema change")) + + // Restarting the query with a new DataFrame should recover from the schema change + q = startQuery() + q.processAllAvailable() + val outputDf = spark.read.format("delta").load(outputDir.getCanonicalPath) + assert(outputDf.schema("id").nullable, + "Expected 'id' column to be nullable after relaxing nullability") + checkAnswer(outputDf.orderBy("id"), + Seq(Row(null), Row("0"), Row("1"))) + } finally { + q.stop() + } + } + } + + test("relax nullability: restarting with stale DataFrame should recover") { + withTempDirs { (inputDir, outputDir, checkpointDir) => + sql(s"CREATE TABLE delta.`${inputDir.getCanonicalPath}` " + + "(id STRING NOT NULL) USING DELTA") + val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) + (0 until 2).foreach { i => + Seq(i.toString).toDF("id") + .write.mode("append").format("delta").save(deltaLog.dataPath.toString) + } + val df = loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) + + var q = df.writeStream + .option("checkpointLocation", checkpointDir.getCanonicalPath) + .option("mergeSchema", "true") + // use delta sink because we need to check the result + .format("delta") + .start(outputDir.getCanonicalPath) + try { + q.processAllAvailable() + + // Clear delta log cache + DeltaLog.clearCache() + // Relax nullability from "id STRING NOT NULL" to "id STRING" using the non-cached + // `DeltaLog` to mimic the case that the schema change happens on a different cluster + withMetadata(deltaLog, StructType.fromDDL("id STRING")) + // Insert a null row + Seq(Option.empty[String]).toDF("id") + .write.mode("append").format("delta").save(deltaLog.dataPath.toString) + + // The streaming query should fail when detecting a schema change + val e = intercept[StreamingQueryException] { + q.processAllAvailable() + } + assert(e.getMessage.contains("Detected schema change")) + + // Restarting the query with the stale DataFrame should still recover + q = df.writeStream + .option("checkpointLocation", checkpointDir.getCanonicalPath) + .option("mergeSchema", "true") + .format("delta") + .start(outputDir.getCanonicalPath) + q.processAllAvailable() + val outputDf = spark.read.format("delta").load(outputDir.getCanonicalPath) + assert(outputDf.schema("id").nullable, + "Expected 'id' column to be nullable after relaxing nullability") + checkAnswer(outputDf.orderBy("id"), + Seq(Row(null), Row("0"), Row("1"))) + } finally { + q.stop() + } + } + } + + test("type widening: restarting with new DataFrame should recover") { + withSQLConf( + DeltaSQLConf.DELTA_TYPE_WIDENING_ENABLE_STREAMING_SCHEMA_TRACKING.key -> "false", + DeltaConfigs.ENABLE_TYPE_WIDENING.defaultTablePropertyKey -> "true", + DeltaSQLConf.DELTA_STREAMING_SINK_ALLOW_IMPLICIT_CASTS.key -> "true") { + withTempDirs { (inputDir, outputDir, checkpointDir) => + sql(s"CREATE TABLE delta.`${inputDir.getCanonicalPath}` (id INT) " + + "USING DELTA TBLPROPERTIES ('delta.enableTypeWidening' = 'true')") + val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) + (0 until 2).foreach { i => + Seq(i).toDF("id") + .write.mode("append").format("delta").save(deltaLog.dataPath.toString) + } + + def startQuery(): StreamingQuery = { + loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) + .writeStream + .option("checkpointLocation", checkpointDir.getCanonicalPath) + .option("mergeSchema", "true") + // use delta sink because we need to check the result + .format("delta") + .start(outputDir.getCanonicalPath) + } + + var q = startQuery() + try { + q.processAllAvailable() + + // Clear delta log cache + DeltaLog.clearCache() + // Widen the column type from INT to BIGINT using the non-cached `DeltaLog` to mimic + // the case that the schema change happens on a different cluster + withMetadata(deltaLog, StructType.fromDDL("id BIGINT")) + // 2^31 cannot be represented as an int, so it will be inserted as a bigint + Seq(2147483648L).toDF("id") + .write.mode("append").format("delta").save(deltaLog.dataPath.toString) + + // The streaming query should fail when detecting a schema change + val e = intercept[StreamingQueryException] { + q.processAllAvailable() + } + assert(e.getMessage.contains("Detected schema change")) + + // Restarting the query with a new DataFrame should recover from the schema change + q = startQuery() + q.processAllAvailable() + val outputDf = spark.read.format("delta").load(outputDir.getCanonicalPath) + assert(outputDf.schema("id").dataType === LongType, + s"Expected 'id' column to be LongType after type widening but got " + + s"${outputDf.schema("id").dataType}") + checkAnswer(outputDf.orderBy("id"), + Seq(Row(0L), Row(1L), Row(2147483648L))) + } finally { + q.stop() + } + } + } + } + + test("type widening: restarting with stale DataFrame should recover") { + withSQLConf( + DeltaSQLConf.DELTA_TYPE_WIDENING_ENABLE_STREAMING_SCHEMA_TRACKING.key -> "false", + DeltaConfigs.ENABLE_TYPE_WIDENING.defaultTablePropertyKey -> "true", + DeltaSQLConf.DELTA_STREAMING_SINK_ALLOW_IMPLICIT_CASTS.key -> "true") { + withTempDirs { (inputDir, outputDir, checkpointDir) => + sql(s"CREATE TABLE delta.`${inputDir.getCanonicalPath}` (id INT) " + + "USING DELTA TBLPROPERTIES ('delta.enableTypeWidening' = 'true')") + val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) + (0 until 2).foreach { i => + Seq(i).toDF("id") + .write.mode("append").format("delta").save(deltaLog.dataPath.toString) + } + val df = loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) + + var q = df.writeStream + .option("checkpointLocation", checkpointDir.getCanonicalPath) + .option("mergeSchema", "true") + // use delta sink because we need to check the result + .format("delta") + .start(outputDir.getCanonicalPath) + try { + q.processAllAvailable() + + // Clear delta log cache + DeltaLog.clearCache() + // Widen the column type from INT to BIGINT using the non-cached `DeltaLog` to mimic + // the case that the schema change happens on a different cluster + withMetadata(deltaLog, StructType.fromDDL("id BIGINT")) + // 2^31 cannot be represented as an int, so it will be inserted as a bigint + Seq(2147483648L).toDF("id") + .write.mode("append").format("delta").save(deltaLog.dataPath.toString) + + // The streaming query should fail when detecting a schema change + val e = intercept[StreamingQueryException] { + q.processAllAvailable() + } + assert(e.getMessage.contains("Detected schema change")) + + // Restarting the query with the stale DataFrame should still recover + q = df.writeStream + .option("checkpointLocation", checkpointDir.getCanonicalPath) + .option("mergeSchema", "true") + .format("delta") + .start(outputDir.getCanonicalPath) + q.processAllAvailable() + val outputDf = spark.read.format("delta").load(outputDir.getCanonicalPath) + assert(outputDf.schema("id").dataType === LongType, + s"Expected 'id' column to be LongType after type widening but got " + + s"${outputDf.schema("id").dataType}") + checkAnswer(outputDf.orderBy("id"), + Seq(Row(0L), Row(1L), Row(2147483648L))) + } finally { + q.stop() + } + } + } + } + + test("drop column: should fail with non-additive schema change error") { + withTempDir { inputDir => + withTempDir { checkpointDir => + // Create a table with column mapping enabled (required for drop/rename column) + sql(s"CREATE TABLE delta.`${inputDir.getCanonicalPath}` (id STRING, value STRING) " + + "USING DELTA " + + "TBLPROPERTIES ('delta.columnMapping.mode' = 'name')") + val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) + (0 until 5).foreach { i => + Seq((i.toString, s"val$i")).toDF("id", "value") + .write.mode("append").format("delta").save(deltaLog.dataPath.toString) + } + + def startQuery(): StreamingQuery = { + loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) + .writeStream + .option("checkpointLocation", checkpointDir.getCanonicalPath) + .format("noop") + .start() + } + + val q = startQuery() + try { + q.processAllAvailable() + + DeltaLog.clearCache() + // Simulate dropping "value" column by committing new metadata with only "id". + // This is a non-additive schema change (column removal). + withMetadata(deltaLog, StructType.fromDDL("id STRING")) + + val e = intercept[StreamingQueryException] { + q.processAllAvailable() + } + assert(e.getMessage.contains( + "Streaming read is not supported on tables with read-incompatible schema changes")) + } finally { + q.stop() + } + } + } + } + + test("drop column: should succeed with unsafe column mapping schema change flag enabled") { + withTempDirs { (inputDir, outputDir, checkpointDir) => + // Enable the unsafe flag that allows streaming to continue past column mapping + // schema changes (drop/rename) instead of failing. + withSQLConf( + DeltaSQLConf + .DELTA_STREAMING_UNSAFE_READ_ON_INCOMPATIBLE_COLUMN_MAPPING_SCHEMA_CHANGES + .key -> "true" + ) { + sql(s"CREATE TABLE delta.`${inputDir.getCanonicalPath}` (id STRING, value STRING) " + + "USING DELTA " + + "TBLPROPERTIES ('delta.columnMapping.mode' = 'name')") + val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) + (0 until 5).foreach { i => + Seq((i.toString, s"val$i")).toDF("id", "value") + .write.mode("append").format("delta").save(deltaLog.dataPath.toString) + } + + val q = loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) + .writeStream + .option("checkpointLocation", checkpointDir.getCanonicalPath) + .format("delta") + .start(outputDir.getCanonicalPath) + try { + q.processAllAvailable() + + DeltaLog.clearCache() + // Simulate dropping "value" column by committing new metadata with only "id". + withMetadata(deltaLog, StructType.fromDDL("id STRING")) + + // Write more data after the drop column schema change + (5 until 10).foreach { i => + Seq(i.toString).toDF("id") + .write.mode("append").format("delta").save(deltaLog.dataPath.toString) + } + + // With the unsafe flag, the stream should process all data without failing. + // Post-drop rows have null for the removed "value" column. + q.processAllAvailable() + + checkAnswer( + spark.read.format("delta").load(outputDir.getAbsolutePath), + (0 until 5).map(i => (i.toString, s"val$i")).toDF("id", "value") union + (5 until 10).map(i => (i.toString, null: String)).toDF("id", "value")) + } finally { + q.stop() + } + } + } + } + + test("rename column: should fail with non-additive schema change error") { + withTempDir { inputDir => + withTempDir { checkpointDir => + sql(s"CREATE TABLE delta.`${inputDir.getCanonicalPath}` (id STRING, value STRING) " + + "USING DELTA " + + "TBLPROPERTIES ('delta.columnMapping.mode' = 'name')") + val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) + (0 until 5).foreach { i => + Seq((i.toString, s"val$i")).toDF("id", "value") + .write.mode("append").format("delta").save(deltaLog.dataPath.toString) + } + + def startQuery(): StreamingQuery = { + loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) + .writeStream + .option("checkpointLocation", checkpointDir.getCanonicalPath) + .format("noop") + .start() + } + + var q = startQuery() + try { + q.processAllAvailable() + + DeltaLog.clearCache() + // Simulate renaming "value" -> "renamed_value" by committing new metadata. + // Column rename is a non-additive schema change under column mapping. + withMetadata(deltaLog, StructType.fromDDL("id STRING, renamed_value STRING")) + + val e = intercept[StreamingQueryException] { + q.processAllAvailable() + } + assert(e.getMessage.contains( + "Streaming read is not supported on tables with read-incompatible schema changes")) + } finally { + q.stop() + } + } + } + } + + test("rename column: should throw schema change error with unsafe flag enabled") { + withTempDir { inputDir => + withTempDir { checkpointDir => + // The unsafe flag only helps with drop-column; rename is still blocked because + // it changes logical column identity, which can silently corrupt data. + withSQLConf( + DeltaSQLConf + .DELTA_STREAMING_UNSAFE_READ_ON_INCOMPATIBLE_COLUMN_MAPPING_SCHEMA_CHANGES + .key -> "true" + ) { + sql(s"CREATE TABLE delta.`${inputDir.getCanonicalPath}` (id STRING, value STRING) " + + "USING DELTA " + + "TBLPROPERTIES ('delta.columnMapping.mode' = 'name')") + val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) + (0 until 5).foreach { i => + Seq((i.toString, s"val$i")).toDF("id", "value") + .write.mode("append").format("delta").save(deltaLog.dataPath.toString) + } + + def startQuery(): StreamingQuery = { + loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) + .writeStream + .option("checkpointLocation", checkpointDir.getCanonicalPath) + .format("noop") + .start() + } + + var q = startQuery() + try { + q.processAllAvailable() + + DeltaLog.clearCache() + // Simulate renaming "value" -> "renamed_value" by committing new metadata. + withMetadata(deltaLog, StructType.fromDDL("id STRING, renamed_value STRING")) + + // Even with the unsafe flag, rename still fails - but with a different error + // ("Detected schema change") rather than the non-additive schema change error. + val e = intercept[StreamingQueryException] { + q.processAllAvailable() + } + assert(e.getMessage.contains("Detected schema change in version 6")) + } finally { + q.stop() + } + } + } + } + } + + test("type widening: should fail with non-additive schema change error " + + "when enable schema tracking") { + withTempDir { inputDir => + withTempDir { checkpointDir => + withSQLConf( + DeltaSQLConf.DELTA_TYPE_WIDENING_ENABLE_STREAMING_SCHEMA_TRACKING.key -> "true" + ) { + // Table with type widening enabled so that type changes are allowed but tracked. + sql(s"CREATE TABLE delta.`${inputDir.getCanonicalPath}` (id INT, value STRING) " + + "USING DELTA " + + "TBLPROPERTIES ('delta.enableTypeWidening' = 'true')") + val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) + (0 until 5).foreach { i => + Seq((i, s"val$i")).toDF("id", "value") + .write.mode("append").format("delta").save(deltaLog.dataPath.toString) + } + + def startQuery(): StreamingQuery = { + loadStreamWithOptions(inputDir.getCanonicalPath, Map.empty) + .writeStream + .option("checkpointLocation", checkpointDir.getCanonicalPath) + .format("noop") + .start() + } + + var q = startQuery() + try { + q.processAllAvailable() + + DeltaLog.clearCache() + // Simulate widening "id" from INT -> BIGINT by committing new metadata. + // Type widening is a non-additive schema change for streaming reads. + withMetadata(deltaLog, StructType.fromDDL("id BIGINT, value STRING")) + + val e = intercept[StreamingQueryException] { + q.processAllAvailable() + } + assert(e.getMessage.contains( + "Streaming read is not supported on tables with read-incompatible schema changes")) + } finally { + q.stop() + } + } + } + } + } + test("handling nullability schema changes") { withTable("srcTable") { withTempDirs { (srcTblDir, checkpointDir, checkpointDir2) => @@ -2374,47 +3000,44 @@ class DeltaSourceSuite extends DeltaSourceSuiteBase } } - test("skip change commits") { - withTempDir { inputDir => - val deltaLog = DeltaLog.forTable(spark, new Path(inputDir.toURI)) - withMetadata(deltaLog, StructType.fromDDL("value STRING")) + test("streaming processes 100 sequential single-value commits and contains all values 0 to 99") { + withTempDirs { (inputDir, outputDir, checkpointDir) => + // TODO(#6339): enable batch size 2 after fix PR merged + assume(!catalogOwnedCoordinatorBackfillBatchSize.contains(2), + "Test cannot pass with batch size 2 due to issue #6339") + // Write the first value to initialize the Delta table + Seq(0).toDF("x").write.format("delta").save(inputDir.toString) - val df = loadStreamWithOptions( - inputDir.getCanonicalPath, - Map(DeltaOptions.SKIP_CHANGE_COMMITS_OPTION -> "true")) + val df = loadStreamWithOptions(inputDir.toString, Map.empty[String, String]) - testStream(df)( - // Add data to source table - AddToReservoir(inputDir, Seq("keep1", "update1", "drop1").toDF()), - AssertOnQuery { q => q.processAllAvailable(); true }, - CheckAnswer("keep1", "update1", "drop1"), - - // Update and delete rows - UpdateReservoir( - inputDir, - Map("value" -> when($"value" === "update1", "updated1").otherwise($"value")) - ), - DeleteFromReservoir(inputDir, $"value" === "drop1"), - CheckAnswer("keep1", "update1", "drop1"), - - // Merge data into source table - MergeIntoReservoir( - inputDir, - dfToMerge = Seq("keep1", "keep2", "keep3").toDF().as("merge1"), - mergeCondition = $"table.value" === $"merge1.value", - Map.empty - ), - MergeIntoReservoir( - inputDir, - dfToMerge = Seq("updated1", "keep4", "keep5").toDF().as("merge2"), - mergeCondition = $"table.value" === $"merge2.value", - Map("table.value" -> when($"table.value" === "updated1", "newlyUpdated1") - .otherwise($"table.value")) - ), - CheckAnswer( - "keep1", "update1", "drop1", "keep2", "keep3" + val q = df.writeStream + .format("delta") + .option("checkpointLocation", checkpointDir.toString) + .start(outputDir.toString) + + try { + // Process the initial commit (value 0) + q.processAllAvailable() + + // Append values 1 through 99, one commit each + (1 until 100).foreach { i => + Seq(i).toDF("x") + .write + .format("delta") + .mode("append") + .save(inputDir.toString) + } + + q.processAllAvailable() + + // Verify the output contains exactly all values from 0 to 99 + checkAnswer( + spark.read.format("delta").load(outputDir.toString), + (0 until 100).map(Row(_)) ) - ) + } finally { + q.stop() + } } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSuite.scala index 0b4070af4f5..68a5d4ba57f 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/DeltaSuite.scala @@ -20,7 +20,10 @@ import java.io.{File, FileNotFoundException} import java.util.concurrent.atomic.AtomicInteger // scalastyle:off import.ordering.noEmptyLine +import org.apache.spark.sql.delta.DeltaOptions +import org.apache.spark.sql.delta.DeltaTestUtils.BOOLEAN_DOMAIN import org.apache.spark.sql.delta.actions.{Action, TableFeatureProtocolUtils} +import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.cdc.CDCReader import org.apache.spark.sql.delta.coordinatedcommits.{CatalogOwnedTableUtils, CatalogOwnedTestBaseSuite} import org.apache.spark.sql.delta.files.TahoeLogFileIndex @@ -37,6 +40,8 @@ import org.apache.spark.SparkException import org.apache.spark.scheduler.{SparkListener, SparkListenerJobStart} import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType} +import org.apache.spark.sql.connector.catalog.TableCatalog import org.apache.spark.sql.catalyst.expressions.InSet import org.apache.spark.sql.catalyst.expressions.Literal.TrueLiteral import org.apache.spark.sql.catalyst.plans.logical.Filter @@ -46,7 +51,7 @@ import org.apache.spark.sql.functions.{asc, col, expr, lit, map_values, struct} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.streaming.StreamingQuery import org.apache.spark.sql.test.SharedSparkSession -import org.apache.spark.sql.types.{StringType, StructType} +import org.apache.spark.sql.types._ import org.apache.spark.util.Utils class DeltaSuite extends QueryTest @@ -241,6 +246,36 @@ class DeltaSuite extends QueryTest } } + test("Cannot create table with NullType UDT column") { + val table_name = "test_table" + withTable(table_name) { + checkError( + intercept[DeltaAnalysisException] { + Seq((1, new NullData())).toDF("id", "value") + .write.format("delta").saveAsTable(table_name) + }, + "DELTA_USER_DEFINED_TYPE_COLUMN_CONTAINS_NULL_TYPE", + sqlState = Some("22005"), + parameters = Map("columnName" -> "value", "userClass" -> classOf[NullData].getName) + ) + } + } + + test("Cannot create table with NullType in a complex UDT column") { + val table_name = "test_table" + withTable(table_name) { + checkError( + intercept[DeltaAnalysisException] { + Seq((1, new ComplexData())).toDF("id", "value") + .write.format("delta").saveAsTable(table_name) + }, + "DELTA_USER_DEFINED_TYPE_COLUMN_CONTAINS_NULL_TYPE", + sqlState = Some("22005"), + parameters = Map("columnName" -> "value", "userClass" -> classOf[ComplexData].getName) + ) + } + } + test("partitioned append - nulls") { val tempDir = Utils.createTempDir() Seq(Some(1), None).toDF() @@ -3012,6 +3047,56 @@ class DeltaSuite extends QueryTest // Not properly supported: ambiguous without special handling for escaping. assert(parseTableAndAlias("'store sales'") === "'store" -> Some("sales'")) } + + test("DeltaTableV2.properties() filters fs.* storage properties injected by catalogs") { + withTempDir { dir => + spark.range(1).write.format("delta").save(dir.getAbsolutePath) + + val tablePath = new Path(dir.toURI) + + // Simulate catalog (e.g., Unity Catalog) injecting fs.* credentials and metadata + // into CatalogTable.storage.properties at table-load time. + val injectedFsProps = Map( + "fs.s3a.fake-endpoint" -> "s3.us-west-2.amazonaws.com", + "fs.unitycatalog.uri" -> "https://uc.example.com", + "fs.unitycatalog.auth.fake-token" -> "dapi_secret_token" + ) + val otherStorageProps = Map( + "nonFsProp" -> "visible_value", + "path" -> dir.getAbsolutePath + ) + val allStorageProps = injectedFsProps ++ otherStorageProps + + val catalogTable = CatalogTable( + identifier = TableIdentifier("test_fs_filter"), + tableType = CatalogTableType.EXTERNAL, + storage = CatalogStorageFormat( + locationUri = Some(dir.toURI), + inputFormat = None, + outputFormat = None, + serde = None, + compressed = false, + properties = allStorageProps + ), + schema = new StructType().add("id", "long"), + provider = Some("delta") + ) + + val deltaTable = DeltaTableV2(spark, tablePath, Some(catalogTable)) + val v2Props = deltaTable.properties() + + injectedFsProps.keys.foreach { fsKey => + assert(!v2Props.containsKey(TableCatalog.OPTION_PREFIX + fsKey), + s"DeltaTableV2.properties() should hide '${TableCatalog.OPTION_PREFIX}$fsKey'") + } + injectedFsProps.keys.foreach { fsKey => + assert(!v2Props.containsKey(fsKey), + s"DeltaTableV2.properties() should also hide '$fsKey'") + } + assert(v2Props.get(TableCatalog.OPTION_PREFIX + "nonFsProp") === "visible_value", + "Non-fs storage properties should remain visible in DeltaTableV2.properties()") + } + } } @@ -3273,6 +3358,69 @@ class DeltaNameColumnMappingSuite extends DeltaSuite insertedDF.filter(col("id") >= 6).union(otherDF)) } } + + for (useNullIntolerantEqualityWithDPO <- BOOLEAN_DOMAIN) { + test(s"useNullIntolerantEqualityWithDPO=$useNullIntolerantEqualityWithDPO " + + "in Dynamic Partition Overwrite") { + withTempDir { tempDir => + Seq[(Option[Int], Option[Int], String)]( + (None, None, "target"), + (Some(1), None, "target"), + (Some(1), Some(2), "target"), + (Some(1), Some(3), "target")) + .toDF("part1", "part2", "row_origin") + .write + .format("delta") + .partitionBy("part1", "part2") + .mode("append") + .save(tempDir.getCanonicalPath) + + Seq[(Option[Int], Option[Int], String)]( + (None, None, "source"), + (Some(1), None, "source"), + (None, Some(1), "source"), + (Some(1), Some(2), "source"), + (Some(2), Some(3), "source")) + .toDF("part1", "part2", "row_origin") + .write + .format("delta") + .partitionBy("part1", "part2") + .mode("overwrite") + .option(DeltaOptions.PARTITION_OVERWRITE_MODE_OPTION, "dynamic") + .option(DeltaOptions.USE_NULL_INTOLERANT_EQUALITY_WITH_DPO, + useNullIntolerantEqualityWithDPO.toString) + .save(tempDir.getCanonicalPath) + + // When this option is set to true, partitions that contain null values are inserted, + // not overwritten. + if (useNullIntolerantEqualityWithDPO) { + checkAnswer( + spark.read.format("delta").load(tempDir.toString), + Row(null, null, "source") :: + Row(null, null, "target") :: + Row(1, null, "source") :: + Row(1, null, "target") :: + Row(null, 1, "source") :: + Row(1, 2, "source") :: + Row(1, 3, "target") :: + Row(2, 3, "source") :: + Nil + ) + } else { + checkAnswer( + spark.read.format("delta").load(tempDir.toString), + Row(null, null, "source") :: + Row(1, null, "source") :: + Row(null, 1, "source") :: + Row(1, 2, "source") :: + Row(1, 3, "target") :: + Row(2, 3, "source") :: + Nil + ) + } + } + } + } } class DeltaWithCatalogOwnedBatch1Suite extends DeltaSuite { @@ -3286,3 +3434,27 @@ class DeltaWithCatalogOwnedBatch2Suite extends DeltaSuite { class DeltaWithCatalogOwnedBatch100Suite extends DeltaSuite { override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(100) } + +@SQLUserDefinedType(udt = classOf[NullUDT]) +class NullData extends Serializable + +class NullUDT extends UserDefinedType[NullData] { + override def sqlType: DataType = NullType + override def userClass: Class[NullData] = classOf[NullData] + override def serialize(obj: NullData): Any = null + override def deserialize(datum: Any): NullData = new NullData() +} + +@SQLUserDefinedType(udt = classOf[ComplexUDT]) +class ComplexData extends Serializable + +class ComplexUDT extends UserDefinedType[ComplexData] { + override def sqlType: DataType = new MapType( + StringType, + new ArrayType( + new StructType().add("a", IntegerType).add("b", new NullUDT), containsNull = true), + valueContainsNull = true) + override def userClass: Class[ComplexData] = classOf[ComplexData] + override def serialize(obj: ComplexData): Any = null + override def deserialize(datum: Any): ComplexData = new ComplexData() +} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/IdentityColumnConflictSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/IdentityColumnConflictSuite.scala index 2bee10029b5..309689583fd 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/IdentityColumnConflictSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/IdentityColumnConflictSuite.scala @@ -134,6 +134,9 @@ trait IdentityColumnConflictSuiteBase None } + /** Executes the winning transaction SQL. Overridable for custom RPC assertions. */ + protected def sqlWithTotalRpcBound(sqlText: String): Unit = sql(sqlText) + /** * Helper function to test two concurrently running commands. Winning transaction commits before * current transaction commits. @@ -157,7 +160,7 @@ trait IdentityColumnConflictSuiteBase unblockUntilPreCommit(txnObserver) busyWaitFor(txnObserver.phases.preparePhase.hasEntered, timeout) - sql(winningTxn.sqlCommand.replace("{tblName}", tblName)) + sqlWithTotalRpcBound(winningTxn.sqlCommand.replace("{tblName}", tblName)) val expectedException = expectedExceptionClass(currentTxn, winningTxn) val events = Log4jUsageLogger.track { diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoStructEvolutionNullnessSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoStructEvolutionNullnessSuite.scala index a9de828e024..5ffa13e24e0 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoStructEvolutionNullnessSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/MergeIntoStructEvolutionNullnessSuite.scala @@ -31,10 +31,10 @@ import org.apache.spark.sql.types.{ArrayType, IntegerType, StructType} trait MergeIntoStructEvolutionNullnessTestUtils extends MergeHelpers { /** Whether to preserve null source structs for struct evolution tests. */ - protected def preserveNullSourceStructs: Boolean + protected def preserveNullSourceStructs: Boolean = true /** Whether to preserve null source structs for UPDATE * specifically. */ - protected def preserveNullSourceStructsUpdateStar: Boolean + protected def preserveNullSourceStructsUpdateStar: Boolean = true /** Configurations for preserving null source structs. */ protected val preserveNullStructsConfs: Seq[(String, String)] = Seq( @@ -2239,21 +2239,3 @@ trait MergeIntoStructEvolutionNullnessMultiClauseTests expectErrorWithoutEvolutionContains = "Cannot cast", confs = preserveNullStructsConfs) } - -trait StructEvolutionPreserveNullSourceEnabled extends MergeIntoStructEvolutionNullnessTestUtils { - override protected def preserveNullSourceStructs: Boolean = true -} - -trait StructEvolutionPreserveNullSourceDisabled extends MergeIntoStructEvolutionNullnessTestUtils { - override protected def preserveNullSourceStructs: Boolean = false -} - -trait StructEvolutionPreserveNullSourceUpdateStarEnabled - extends MergeIntoStructEvolutionNullnessTestUtils { - override protected def preserveNullSourceStructsUpdateStar: Boolean = true -} - -trait StructEvolutionPreserveNullSourceUpdateStarDisabled - extends MergeIntoStructEvolutionNullnessTestUtils { - override protected def preserveNullSourceStructsUpdateStar: Boolean = false -} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/OptimisticTransactionSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/OptimisticTransactionSuite.scala index 87fd5c2ae62..b723e9fc54d 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/OptimisticTransactionSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/OptimisticTransactionSuite.scala @@ -1198,16 +1198,22 @@ class OptimisticTransactionSuite val logRecords = Log4jUsageLogger.track { exceptionClassOpt match { - case Some(exceptionClass) => - val ex = intercept[Throwable] { - txn.commit( - Seq(newMetadata), - DeltaOperations.Update(predicate = Some(EqualTo(Literal(1), Literal(1)))) + case Some(_) => + checkError( + intercept[DeltaAnalysisException] { + txn.commit( + Seq(newMetadata), + DeltaOperations.Update(predicate = Some(EqualTo(Literal(1), Literal(1)))) + ) + }, + condition = "DELTA_UNSUPPORTED_PARTITION_COLUMN_CHANGE", + sqlState = "42P10", + parameters = Map( + "operation" -> "UPDATE", + "oldPartitionColumns" -> "col1", + "newPartitionColumns" -> "col2" ) - } - assert(ex.getClass == exceptionClass) - assert(ex.asInstanceOf[DeltaAnalysisException].getErrorClass == - "DELTA_UNSUPPORTED_PARTITION_COLUMN_CHANGE") + ) case None => // Should succeed without throwing txn.commit( @@ -1245,41 +1251,49 @@ class OptimisticTransactionSuite val txn = deltaLog.startTransaction() val newMetadata = txn.metadata.copy(partitionColumns = Seq("col2")) - val ex = intercept[DeltaAnalysisException] { - txn.commit( - Seq(newMetadata), - DeltaOperations.Update(predicate = Some(EqualTo(Literal(1), Literal(1)))) + checkError( + intercept[DeltaAnalysisException] { + txn.commit( + Seq(newMetadata), + DeltaOperations.Update(predicate = Some(EqualTo(Literal(1), Literal(1)))) + ) + }, + condition = "DELTA_UNSUPPORTED_PARTITION_COLUMN_CHANGE", + sqlState = "42P10", + parameters = Map( + "operation" -> "UPDATE", + "oldPartitionColumns" -> "col1", + "newPartitionColumns" -> "col2" ) - } - assert(ex.getErrorClass == "DELTA_UNSUPPORTED_PARTITION_COLUMN_CHANGE") + ) assertPartitionColumns(tablePath, Seq("col1")) } } Seq( // Recreation of running DFv1 .save() overwrite changing partition cols. - ("blocked for Write(Overwrite, partitionBy)", + ("blocked for Write(Overwrite, partitionBy)", "WRITE", (_: Metadata) => DeltaOperations.Write(SaveMode.Overwrite, partitionBy = Some(Seq("col2")))), // Recreation of running DFv1 .save() replaceWhere changing partition cols. - ("blocked for Write(Overwrite, partitionBy, predicate)", + ("blocked for Write(Overwrite, partitionBy, predicate)", "WRITE", (_: Metadata) => DeltaOperations.Write(SaveMode.Overwrite, partitionBy = Some(Seq("col2")), predicate = Some("col1=0"))), // Recreation of running DFv1 .save() DPO changing partition cols. - ("blocked for Write(Overwrite, partitionBy, DPO)", + ("blocked for Write(Overwrite, partitionBy, DPO)", "WRITE", (_: Metadata) => DeltaOperations.Write(SaveMode.Overwrite, partitionBy = Some(Seq("col2")), isDynamicPartitionOverwrite = Some(true))), // Recreation of running DFv1 .saveAsTable() overwrite changing partition cols. - ("blocked for ReplaceTable(isV1SaveAsTableOverwrite=true)", + ("blocked for ReplaceTable(isV1SaveAsTableOverwrite=true)", "CREATE OR REPLACE TABLE AS SELECT", (newMeta: Metadata) => DeltaOperations.ReplaceTable( metadata = newMeta, isManaged = true, orCreate = true, asSelect = true, isV1SaveAsTableOverwrite = Some(true))) - ).foreach { case (testSuffix, mkOp) => + ).foreach { case (testSuffix, expectedOpName, mkOp) => test(s"partition column changes $testSuffix") { withTempDir { tempDir => val tablePath = tempDir.getAbsolutePath @@ -1289,10 +1303,18 @@ class OptimisticTransactionSuite val txn = deltaLog.startTransaction() val newMetadata = txn.metadata.copy(partitionColumns = Seq("col2")) - val ex = intercept[DeltaAnalysisException] { - txn.commit(Seq(newMetadata), mkOp(newMetadata)) - } - assert(ex.getErrorClass == "DELTA_UNSUPPORTED_PARTITION_COLUMN_CHANGE") + checkError( + intercept[DeltaAnalysisException] { + txn.commit(Seq(newMetadata), mkOp(newMetadata)) + }, + condition = "DELTA_UNSUPPORTED_PARTITION_COLUMN_CHANGE", + sqlState = "42P10", + parameters = Map( + "operation" -> expectedOpName, + "oldPartitionColumns" -> "col1", + "newPartitionColumns" -> "col2" + ) + ) assertPartitionColumns(tablePath, Seq("col1")) } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/UCManagedTableKillSwitchSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/UCManagedTableKillSwitchSuite.scala new file mode 100644 index 00000000000..6a285829a13 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/UCManagedTableKillSwitchSuite.scala @@ -0,0 +1,91 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta + +import org.apache.spark.sql.delta.actions.DomainMetadata +import org.apache.spark.sql.delta.coordinatedcommits.CatalogOwnedTestBaseSuite +import org.apache.spark.sql.delta.skipping.clustering.{ClusteredTableUtils, ClusteringColumn} +import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils} + +import org.apache.spark.sql.catalyst.TableIdentifier + +/** + * Unit tests for the kill switch that blocks clustering column changes on UC-managed + * CatalogOwned tables. These tests bypass UCSingleCatalog (only available in sparkUnityCatalog) + * by overriding [[OptimisticTransaction.isUCManagedTable]] in a test-local subclass. + * + * The kill switch fires in [[OptimisticTransaction.commitLarge]], which is used by RESTORE TABLE + * and bypasses prepareCommit. + */ +class UCManagedTableKillSwitchSuite + extends CatalogOwnedTestBaseSuite + with DeltaSQLTestUtils + with DeltaSQLCommandTest { + + // Enable CatalogOwned by default so every CREATE TABLE produces a CatalogOwned table. + override def catalogOwnedCoordinatorBackfillBatchSize: Option[Int] = Some(1) + + /** + * Test subclass that pretends it targets a UC-managed table, bypassing the real + * [[CatalogOwnedTableUtils.getCatalogName]] check that requires UCSingleCatalog. + */ + private class UCManagedTxn(log: DeltaLog, snap: Snapshot) + extends OptimisticTransaction(log, None, snap) { + override protected[delta] lazy val isUCManagedTable: Boolean = true + } + + private val clusteringOnId: Seq[DomainMetadata] = + Seq(ClusteredTableUtils.createDomainMetadata(Seq(ClusteringColumn(Seq("id"))))) + + private val clusteringOnName: Seq[DomainMetadata] = + Seq(ClusteredTableUtils.createDomainMetadata(Seq(ClusteringColumn(Seq("name"))))) + + /** Creates a CatalogOwned clustered-by-id table and returns its (log, snapshot). */ + private def createClusteredTable(tableName: String): (DeltaLog, Snapshot) = { + sql(s"CREATE TABLE $tableName (id INT, name STRING) USING delta " + + s"CLUSTER BY (id) TBLPROPERTIES ('delta.feature.catalogManaged' = 'supported')") + sql(s"INSERT INTO $tableName VALUES (1, 'a')") + val (log, snap) = + DeltaLog.forTableWithSnapshot(spark, TableIdentifier(tableName)) + assert(snap.isCatalogOwned, "table should be CatalogOwned") + (log, snap) + } + + test("commitLarge blocks clustering change on UC-managed CatalogOwned table") { + withTable("tbl") { + val (log, snap) = createClusteredTable("tbl") + val ex = intercept[DeltaAnalysisException] { + new UCManagedTxn(log, snap).commitLarge( + spark, + clusteringOnName.iterator, + newProtocolOpt = None, + op = DeltaOperations.Restore(Some(0L), None), + context = Map.empty, + metrics = Map.empty) + } + assert(ex.getMessage.contains("Clustering column changes on Unity Catalog managed tables")) + } + } + + test("commit with unchanged clustering is allowed on UC-managed CatalogOwned table") { + withTable("tbl") { + val (log, snap) = createClusteredTable("tbl") + // Commit the same clustering DomainMetadata that the snapshot already has - must not throw. + new UCManagedTxn(log, snap).commit(clusteringOnId, DeltaOperations.ManualUpdate) + } + } +} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/UniversalFormatSuiteBase.scala b/spark/src/test/scala/org/apache/spark/sql/delta/UniversalFormatSuiteBase.scala index c81dec7a2f8..625ad4a77a8 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/UniversalFormatSuiteBase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/UniversalFormatSuiteBase.scala @@ -445,12 +445,14 @@ trait UniversalFormatMiscSuiteBase extends IcebergCompatUtilsBase with Universal ) updatedConfiguration = getUpdatedConfiguration(configurationUnderTest) - assert(updatedConfiguration.size == 5) + assert(updatedConfiguration.size == 6) assert(updatedConfiguration("dummykey") == "dummyvalue") assert(updatedConfiguration("delta.universalFormat.enabledFormats") == "iceberg") assert(updatedConfiguration("delta.columnMapping.mode") == "name") assert(updatedConfiguration(s"delta.enableIcebergCompatV$icv") == "true") assert(updatedConfiguration("delta.columnMapping.maxColumnId") == "1") + assert(updatedConfiguration( + DeltaConfigs.ICEBERG_ATOMIC_CONVERSION_SUPPORTED.key) == "true") configurationUnderTest = Map( s"delta.enableIcebergCompatV$icv" -> "true", @@ -459,7 +461,10 @@ trait UniversalFormatMiscSuiteBase extends IcebergCompatUtilsBase with Universal "delta.columnMapping.mode" -> "id" ) updatedConfiguration = getUpdatedConfiguration(configurationUnderTest) - assert(updatedConfiguration.size == 4) + assert(updatedConfiguration.size == 6) + assert(updatedConfiguration("delta.columnMapping.maxColumnId") == "1") + assert(updatedConfiguration( + DeltaConfigs.ICEBERG_ATOMIC_CONVERSION_SUPPORTED.key) == "true") assert(updatedConfiguration("dummykey") == "dummyvalue") assert(updatedConfiguration("delta.columnMapping.mode") == "id") assert(updatedConfiguration("delta.universalFormat.enabledFormats") == "iceberg") @@ -468,6 +473,38 @@ trait UniversalFormatMiscSuiteBase extends IcebergCompatUtilsBase with Universal } } + test("enforceDependenciesInConfiguration preserves existing atomic guard property") { + withTempTableAndDir { case (id, _) => + executeSql(s"CREATE TABLE $id (id INT) USING DELTA") + val catalogTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(id)) + + def getConfig: Map[String, String] = { + val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, TableIdentifier(id)) + UniversalFormat.enforceDependenciesInConfiguration( + spark, catalogTable, + Map(DeltaConfigs.ICEBERG_COMPAT_V2_ENABLED.key -> "true"), + snapshot + ) + } + + // Case 1: snapshot has no guard -> auto-set to "true" + assert(getConfig.get( + DeltaConfigs.ICEBERG_ATOMIC_CONVERSION_SUPPORTED.key).contains("true")) + + // Case 2: snapshot has guard = false -> preserve false + executeSql(s"ALTER TABLE $id SET TBLPROPERTIES " + + s"('${DeltaConfigs.ICEBERG_ATOMIC_CONVERSION_SUPPORTED.key}' = 'false')") + assert(getConfig.get( + DeltaConfigs.ICEBERG_ATOMIC_CONVERSION_SUPPORTED.key).contains("false")) + + // Case 3: snapshot has guard = true -> preserve true + executeSql(s"ALTER TABLE $id SET TBLPROPERTIES " + + s"('${DeltaConfigs.ICEBERG_ATOMIC_CONVERSION_SUPPORTED.key}' = 'true')") + assert(getConfig.get( + DeltaConfigs.ICEBERG_ATOMIC_CONVERSION_SUPPORTED.key).contains("true")) + } + } + test("UniForm config validation") { Seq("ICEBERG", "iceberg,iceberg", "iceber", "paimon").foreach { invalidConf => withTempTableAndDir { case (id, loc) => diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/VersionChecksumHistogramCompatSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/VersionChecksumHistogramCompatSuite.scala index 69b5dc67402..e7725fa9431 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/VersionChecksumHistogramCompatSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/VersionChecksumHistogramCompatSuite.scala @@ -16,6 +16,8 @@ package org.apache.spark.sql.delta +import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.delta.stats.FileSizeHistogram import org.apache.spark.sql.delta.test.DeltaSQLCommandTest import org.apache.spark.sql.delta.util.{FileNames, JsonUtils} @@ -267,4 +269,83 @@ class VersionChecksumHistogramCompatSuite "Scenario B: fileSizeHistogram (last in JSON) should win over histogramOpt") } } + + + test("writeChecksumFile writes correct field name based on conf") { + withTempDir { dir => + spark.range(10).write.format("delta").save(dir.getAbsolutePath) + val deltaLog = DeltaLog.forTable(spark, dir.getAbsolutePath) + val testHistogram = FileSizeHistogram( + sortedBinBoundaries = Vector(0L, 1024L, 10240L), + fileCounts = Array(5L, 10L, 15L), + totalBytes = Array(100L, 200L, 300L)) + val checksum = deltaLog.snapshot.checksumOpt.get.copy(histogramOpt = Some(testHistogram)) + + val currentSpark = spark + val currentLog = deltaLog + val writer = new RecordChecksum { + override val deltaLog: DeltaLog = currentLog + override protected def spark: org.apache.spark.sql.SparkSession = currentSpark + def writeChecksum(version: Long, cs: VersionChecksum): Unit = + writeChecksumFile(version, cs) + } + + // Write with flag OFF (default) -- should use histogramOpt + val versionOff = deltaLog.snapshot.version + 1 + withSQLConf(DeltaSQLConf.DELTA_CHECKSUM_HISTOGRAM_FIELD_FOLLOWS_PROTOCOL.key -> "false") { + writer.writeChecksum(versionOff, checksum) + } + val crcJsonOff = + deltaLog.store.read(FileNames.checksumFile(deltaLog.logPath, versionOff)).head + assert(crcJsonOff.contains("\"histogramOpt\":"), + "Flag OFF: CRC should contain histogramOpt") + assert(!crcJsonOff.contains("\"fileSizeHistogram\":"), + "Flag OFF: CRC should not contain fileSizeHistogram") + + // Write with flag ON -- should use fileSizeHistogram + val versionOn = versionOff + 1 + withSQLConf(DeltaSQLConf.DELTA_CHECKSUM_HISTOGRAM_FIELD_FOLLOWS_PROTOCOL.key -> "true") { + writer.writeChecksum(versionOn, checksum) + } + val crcJsonOn = + deltaLog.store.read(FileNames.checksumFile(deltaLog.logPath, versionOn)).head + assert(crcJsonOn.contains("\"fileSizeHistogram\":"), + "Flag ON: CRC should contain fileSizeHistogram") + assert(!crcJsonOn.contains("\"histogramOpt\":"), + "Flag ON: CRC should not contain histogramOpt") + + // Both CRCs should be readable and produce the same histogram + val checksumOff = JsonUtils.mapper.readValue[VersionChecksum](crcJsonOff) + val checksumOn = JsonUtils.mapper.readValue[VersionChecksum](crcJsonOn) + assert(checksumOff.histogramOpt.get === testHistogram, + "Flag OFF: read-back histogram should match the test histogram") + assert(checksumOn.histogramOpt.get === testHistogram, + "Flag ON: read-back histogram should match the test histogram") + } + } + + test("VersionChecksumProtocolCompliant fields match VersionChecksum") { + // Use reflection to ensure the two classes stay in sync. If someone adds a field to + // VersionChecksum but forgets VersionChecksumProtocolCompliant, this test will catch it. + val checksumFields = classOf[VersionChecksum].getDeclaredFields + .map(f => (f.getName, f.getType)).toSet + val protocolCompliantFields = classOf[VersionChecksumProtocolCompliant].getDeclaredFields + .map(f => (f.getName, f.getType)).toSet + + // The only difference should be histogramOpt vs fileSizeHistogram (same type) + val expectedOnlyInChecksum = Set(("histogramOpt", classOf[Option[_]])) + val expectedOnlyInProtocolCompliant = Set(("fileSizeHistogram", classOf[Option[_]])) + + val onlyInChecksum = checksumFields -- protocolCompliantFields + val onlyInProtocolCompliant = protocolCompliantFields -- checksumFields + + assert(onlyInChecksum === expectedOnlyInChecksum, + s"Unexpected fields only in VersionChecksum: $onlyInChecksum. " + + "Did you add a new field to VersionChecksum without updating " + + "VersionChecksumProtocolCompliant?") + assert(onlyInProtocolCompliant === expectedOnlyInProtocolCompliant, + s"Unexpected fields only in VersionChecksumProtocolCompliant: $onlyInProtocolCompliant. " + + "Did you add a new field to VersionChecksumProtocolCompliant without updating " + + "VersionChecksum?") + } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/concurrency/TransactionExecutionTestMixin.scala b/spark/src/test/scala/org/apache/spark/sql/delta/concurrency/TransactionExecutionTestMixin.scala index 059314aa072..2b1a83e6850 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/concurrency/TransactionExecutionTestMixin.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/concurrency/TransactionExecutionTestMixin.scala @@ -53,6 +53,7 @@ trait TransactionExecutionTestMixin { spark.withActive( try { TransactionExecutionObserver.withObserver(observer) { + observer.phases.analysisPhase.waitToEnter() fn() } } catch { @@ -114,6 +115,7 @@ trait TransactionExecutionTestMixin { /** Unblocks all phases before the `commitPhase` for [[TransactionObserver]] */ def unblockUntilPreCommit(observer: TransactionObserver): Unit = { + observer.phases.analysisPhase.entryBarrier.unblock() observer.phases.initialPhase.entryBarrier.unblock() observer.phases.preparePhase.entryBarrier.unblock() } @@ -129,6 +131,7 @@ trait TransactionExecutionTestMixin { /** Unblocks all phases for [[TransactionObserver]] so that corresponding query can finish. */ def unblockAllPhases(observer: TransactionObserver): Unit = { + observer.phases.analysisPhase.entryBarrier.unblock() observer.phases.initialPhase.entryBarrier.unblock() observer.phases.preparePhase.entryBarrier.unblock() observer.phases.commitPhase.entryBarrier.unblock() diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CatalogOwnedPropertySuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CatalogOwnedPropertySuite.scala index 87324dc4db1..a1cc206e5ad 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CatalogOwnedPropertySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/coordinatedcommits/CatalogOwnedPropertySuite.scala @@ -116,6 +116,11 @@ class CatalogOwnedPropertySuite extends QueryTest ucTableId.get } + private def getSnapshotVersion(tableName: String): Long = { + val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, new TableIdentifier(tableName)) + snapshot.version + } + private def validateInCommitTimestampTableFeature(tableName: String, expected: Boolean): Unit = { val (_, snapshot) = DeltaLog.forTableWithSnapshot(spark, new TableIdentifier(tableName)) val writerFeatureNames = snapshot.protocol.writerFeatureNames @@ -241,6 +246,7 @@ class CatalogOwnedPropertySuite extends QueryTest // CatalogManaged enabled table. createTableAndValidateCatalogOwned(tableName = "t1", withCatalogOwned = true) val ucTableIdBefore = getUCTableIdFromTable(tableName = "t1") + val versionBefore = getSnapshotVersion(tableName = "t1") // Specifying CatalogManaged on an already CatalogManaged table should succeed. // The CatalogManaged properties are treated as a no-op. @@ -251,6 +257,7 @@ class CatalogOwnedPropertySuite extends QueryTest validateCatalogOwnedAndUCTableId(tableName = "t1", expected = true) val ucTableIdAfter = getUCTableIdFromTable(tableName = "t1") assert(ucTableIdBefore === ucTableIdAfter) + assert(getSnapshotVersion(tableName = "t1") === versionBefore + 1) } } @@ -280,6 +287,7 @@ class CatalogOwnedPropertySuite extends QueryTest // CatalogManaged enabled table. createTableAndValidateCatalogOwned(tableName = "t1", withCatalogOwned = true) val ucTableIdBefore = getUCTableIdFromTable(tableName = "t1") + val versionBefore = getSnapshotVersion(tableName = "t1") // Source RTAS table. createTableAndValidateCatalogOwned(tableName = "t2", withCatalogOwned = false) @@ -294,6 +302,48 @@ class CatalogOwnedPropertySuite extends QueryTest validateCatalogOwnedAndUCTableId(tableName = "t1", expected = true) val ucTableIdAfter = getUCTableIdFromTable(tableName = "t1") assert(ucTableIdBefore === ucTableIdAfter) + assert(getSnapshotVersion(tableName = "t1") === versionBefore + 1) + checkAnswer(sql("SELECT * FROM t1"), Seq(Row(1), Row(2))) + } + } + + test("[RTAS] failed replace preserves existing catalog-managed table data and version") { + withTable("t1") { + createTableAndValidateCatalogOwned(tableName = "t1", withCatalogOwned = true) + val ucTableIdBefore = getUCTableIdFromTable(tableName = "t1") + sql("INSERT INTO t1 VALUES (1), (2)") + val versionBefore = getSnapshotVersion(tableName = "t1") + + intercept[Exception] { + sql( + """REPLACE TABLE t1 USING delta AS + |SELECT IF(id = 2L, CAST(raise_error('boom') AS BIGINT), id) AS id + |FROM VALUES (1L), (2L) AS src(id) + |""".stripMargin) + } + + validateCatalogOwnedAndUCTableId(tableName = "t1", expected = true) + assert(getUCTableIdFromTable(tableName = "t1") === ucTableIdBefore) + assert(getSnapshotVersion(tableName = "t1") === versionBefore) + checkAnswer(sql("SELECT * FROM t1"), Seq(Row(1), Row(2))) + } + } + + test("[RTAS] replacing an existing catalog-managed table preserves UC identity") { + withTable("t1", "t2") { + createTableAndValidateCatalogOwned(tableName = "t1", withCatalogOwned = true) + val ucTableIdBefore = getUCTableIdFromTable(tableName = "t1") + sql("INSERT INTO t1 VALUES (10)") + val versionBefore = getSnapshotVersion(tableName = "t1") + + createTableAndValidateCatalogOwned(tableName = "t2", withCatalogOwned = false) + sql("INSERT INTO t2 VALUES (1), (2)") + + sql("REPLACE TABLE t1 USING delta AS SELECT * FROM t2") + + validateCatalogOwnedAndUCTableId(tableName = "t1", expected = true) + assert(getUCTableIdFromTable(tableName = "t1") === ucTableIdBefore) + assert(getSnapshotVersion(tableName = "t1") === versionBefore + 1) checkAnswer(sql("SELECT * FROM t1"), Seq(Row(1), Row(2))) } } @@ -319,6 +369,7 @@ class CatalogOwnedPropertySuite extends QueryTest createTableAndValidateCatalogOwned(tableName = "t1", withCatalogOwned = true) val ucTableIdBefore = getUCTableIdFromTable(tableName = "t1") sql("INSERT INTO t1 VALUES (1)") + val versionBefore = getSnapshotVersion(tableName = "t1") // CREATE OR REPLACE with CatalogManaged on existing CatalogManaged table should succeed. sql("CREATE OR REPLACE TABLE t1 (id LONG) USING delta TBLPROPERTIES " + @@ -328,6 +379,65 @@ class CatalogOwnedPropertySuite extends QueryTest validateCatalogOwnedAndUCTableId(tableName = "t1", expected = true) val ucTableIdAfter = getUCTableIdFromTable(tableName = "t1") assert(ucTableIdBefore === ucTableIdAfter) + assert(getSnapshotVersion(tableName = "t1") === versionBefore + 1) + } + } + + test("[CREATE OR REPLACE] replacing an existing catalog-managed table is atomic") { + withTable("t1") { + createTableAndValidateCatalogOwned(tableName = "t1", withCatalogOwned = true) + val ucTableIdBefore = getUCTableIdFromTable(tableName = "t1") + sql("INSERT INTO t1 VALUES (1)") + val versionBefore = getSnapshotVersion(tableName = "t1") + + sql("CREATE OR REPLACE TABLE t1 USING delta TBLPROPERTIES " + + s"('delta.feature.${CatalogOwnedTableFeature.name}' = 'supported') AS " + + "SELECT 2 AS id") + + validateCatalogOwnedAndUCTableId(tableName = "t1", expected = true) + assert(getUCTableIdFromTable(tableName = "t1") === ucTableIdBefore) + assert(getSnapshotVersion(tableName = "t1") === versionBefore + 1) + checkAnswer(sql("SELECT * FROM t1"), Seq(Row(2))) + } + } + + test("[REPLACE] replacing an existing catalog-managed table preserves UC identity") { + withTable("t1") { + createTableAndValidateCatalogOwned(tableName = "t1", withCatalogOwned = true) + val ucTableIdBefore = getUCTableIdFromTable(tableName = "t1") + sql("INSERT INTO t1 VALUES (1)") + val versionBefore = getSnapshotVersion(tableName = "t1") + + sql("REPLACE TABLE t1 (id LONG) USING delta") + + validateCatalogOwnedAndUCTableId(tableName = "t1", expected = true) + assert(getUCTableIdFromTable(tableName = "t1") === ucTableIdBefore) + assert(getSnapshotVersion(tableName = "t1") === versionBefore + 1) + checkAnswer(sql("SELECT * FROM t1"), Seq.empty) + } + } + + test("[CREATE OR REPLACE] specifying table UUID for existing catalog-managed table " + + "should be blocked") { + withTable("t1") { + createTableAndValidateCatalogOwned(tableName = "t1", withCatalogOwned = true) + val ucTableIdBefore = getUCTableIdFromTable(tableName = "t1") + sql("INSERT INTO t1 VALUES (1)") + val versionBefore = getSnapshotVersion(tableName = "t1") + + val error = intercept[DeltaUnsupportedOperationException] { + sql("CREATE OR REPLACE TABLE t1 USING delta TBLPROPERTIES " + + s"('delta.feature.${CatalogOwnedTableFeature.name}' = 'supported', " + + s"'${UCCommitCoordinatorClient.UC_TABLE_ID_KEY}' = '${UUID.randomUUID().toString}') " + + "AS SELECT 2 AS id") + } + checkError(error, "DELTA_CANNOT_MODIFY_TABLE_PROPERTY", "42939", + Map("prop" -> "io.unitycatalog.tableId")) + + validateCatalogOwnedAndUCTableId(tableName = "t1", expected = true) + assert(getUCTableIdFromTable(tableName = "t1") === ucTableIdBefore) + assert(getSnapshotVersion(tableName = "t1") === versionBefore) + checkAnswer(sql("SELECT * FROM t1"), Seq(Row(1))) } } @@ -352,7 +462,7 @@ class CatalogOwnedPropertySuite extends QueryTest } } - test("[CREATE OR REPLACE] continuous CREATE OR REPLACE with CatalogManaged " + + test("[CREATE OR REPLACE] repeated same-schema CREATE OR REPLACE with CatalogManaged " + "should succeed repeatedly") { withTable("t1") { // First CREATE OR REPLACE creates the table. @@ -372,16 +482,16 @@ class CatalogOwnedPropertySuite extends QueryTest val ucTableIdSecond = getUCTableIdFromTable(tableName = "t1") assert(ucTableIdFirst === ucTableIdSecond) - // Third CREATE OR REPLACE should also succeed. - sql("CREATE OR REPLACE TABLE t1 (id LONG, name STRING) USING delta TBLPROPERTIES " + + // Third CREATE OR REPLACE should also succeed when the metadata is unchanged. + sql("CREATE OR REPLACE TABLE t1 (id LONG) USING delta TBLPROPERTIES " + s"('delta.feature.${CatalogOwnedTableFeature.name}' = 'supported')") validateCatalogOwnedAndUCTableId(tableName = "t1", expected = true) val ucTableIdThird = getUCTableIdFromTable(tableName = "t1") assert(ucTableIdFirst === ucTableIdThird) // Verify table is functional. - sql("INSERT INTO t1 VALUES (1, 'a'), (2, 'b')") - checkAnswer(sql("SELECT * FROM t1"), Seq(Row(1, "a"), Row(2, "b"))) + sql("INSERT INTO t1 VALUES (2)") + checkAnswer(sql("SELECT * FROM t1"), Seq(Row(2))) } } test("[CREATE LIKE] Specifying table UUID should be blocked") { diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoNestedArrayStructEvolutionNullnessTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoNestedArrayStructEvolutionNullnessTests.scala index 60499b46e2a..7c7218f1354 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoNestedArrayStructEvolutionNullnessTests.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoNestedArrayStructEvolutionNullnessTests.scala @@ -32,102 +32,7 @@ import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ -class MergeIntoNestedArrayStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOffPreservG4SCW5ASuite +class MergeIntoNestedArrayStructEvolutionNullnessSQLNameBasedSuite extends MergeIntoNestedArrayStructEvolutionNullnessTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedArrayStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOffPreservO4GSSLISuite - extends MergeIntoNestedArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedArrayStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOnPreserveEF65NXYSuite - extends MergeIntoNestedArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedArrayStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOnPreserveUXQCW4ISuite - extends MergeIntoNestedArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedArrayStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOffPrese5QFIOCISuite - extends MergeIntoNestedArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedArrayStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOffPrese7LUJGJASuite - extends MergeIntoNestedArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedArrayStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOnPreser2UCWDCASuite - extends MergeIntoNestedArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedArrayStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOnPreserY6M56VASuite - extends MergeIntoNestedArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedArrayStructEvolutionNullnessSQLNameBasedPreserveNullSourceOffPreserveNullSourceU6MQ3SIISuite - extends MergeIntoNestedArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - -class MergeIntoNestedArrayStructEvolutionNullnessSQLNameBasedPreserveNullSourceOffPreserveNullSourceUEG2RWVYSuite - extends MergeIntoNestedArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - -class MergeIntoNestedArrayStructEvolutionNullnessSQLNameBasedPreserveNullSourceOnPreserveNullSourceUpFQ7PINASuite - extends MergeIntoNestedArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - -class MergeIntoNestedArrayStructEvolutionNullnessSQLNameBasedPreserveNullSourceOnPreserveNullSourceUpR36OX5ISuite - extends MergeIntoNestedArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoNestedMapStructEvolutionNullnessTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoNestedMapStructEvolutionNullnessTests.scala index 32904042286..8b07e74679b 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoNestedMapStructEvolutionNullnessTests.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoNestedMapStructEvolutionNullnessTests.scala @@ -32,102 +32,7 @@ import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ -class MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOffPreserveN4NGDUBYSuite +class MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedSuite extends MergeIntoNestedMapStructEvolutionNullnessTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOffPreserveND64DRCASuite - extends MergeIntoNestedMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOnPreserveNuG4OZSQISuite - extends MergeIntoNestedMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOnPreserveNuQSZI2SISuite - extends MergeIntoNestedMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOffPreservUKDCWWISuite - extends MergeIntoNestedMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOffPreservVUMRVOQSuite - extends MergeIntoNestedMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOnPreserveKMOZRLISuite - extends MergeIntoNestedMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOnPreserveWJIHVZYSuite - extends MergeIntoNestedMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedPreserveNullSourceOffPreserveNullSourceUpd7CQVRRQSuite - extends MergeIntoNestedMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - -class MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedPreserveNullSourceOffPreserveNullSourceUpdI3MWPXASuite - extends MergeIntoNestedMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - -class MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedPreserveNullSourceOnPreserveNullSourceUpda5FZ34QYSuite - extends MergeIntoNestedMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - -class MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedPreserveNullSourceOnPreserveNullSourceUpdaH76RPFYSuite - extends MergeIntoNestedMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoNestedStructEvolutionNullnessTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoNestedStructEvolutionNullnessTests.scala index cc5f3e5035a..2031b538824 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoNestedStructEvolutionNullnessTests.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoNestedStructEvolutionNullnessTests.scala @@ -32,102 +32,7 @@ import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ -class MergeIntoNestedStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOffPreserveNullXPVWNHQSuite +class MergeIntoNestedStructEvolutionNullnessSQLNameBasedSuite extends MergeIntoNestedStructEvolutionNullnessTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOffPreserveNullY6JU3TISuite - extends MergeIntoNestedStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOnPreserveNullS6EZ266ISuite - extends MergeIntoNestedStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOnPreserveNullSMTAVKOYSuite - extends MergeIntoNestedStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOffPreserveNu3TOZDYYSuite - extends MergeIntoNestedStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOffPreserveNu6WJICSYSuite - extends MergeIntoNestedStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOnPreserveNulF3SBMJASuite - extends MergeIntoNestedStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOnPreserveNulS5ZNPDISuite - extends MergeIntoNestedStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoNestedStructEvolutionNullnessSQLNameBasedPreserveNullSourceOffPreserveNullSourceUpdateStarOffSuite - extends MergeIntoNestedStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - -class MergeIntoNestedStructEvolutionNullnessSQLNameBasedPreserveNullSourceOffPreserveNullSourceUpdateStarOnSuite - extends MergeIntoNestedStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - -class MergeIntoNestedStructEvolutionNullnessSQLNameBasedPreserveNullSourceOnPreserveNullSourceUpdateStarOffSuite - extends MergeIntoNestedStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - -class MergeIntoNestedStructEvolutionNullnessSQLNameBasedPreserveNullSourceOnPreserveNullSourceUpdateStarOnSuite - extends MergeIntoNestedStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoStructEvolutionNullnessMultiClauseTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoStructEvolutionNullnessMultiClauseTests.scala index 27d18cea045..4fd48cec149 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoStructEvolutionNullnessMultiClauseTests.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoStructEvolutionNullnessMultiClauseTests.scala @@ -32,102 +32,7 @@ import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ -class MergeIntoStructEvolutionNullnessMultiClauseSQLNameBasedColMapIdModePreserveNullSourceOffPreserv23BA7GQSuite +class MergeIntoStructEvolutionNullnessMultiClauseSQLNameBasedSuite extends MergeIntoStructEvolutionNullnessMultiClauseTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoStructEvolutionNullnessMultiClauseSQLNameBasedColMapIdModePreserveNullSourceOffPreservUYSKTYQSuite - extends MergeIntoStructEvolutionNullnessMultiClauseTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoStructEvolutionNullnessMultiClauseSQLNameBasedColMapIdModePreserveNullSourceOnPreserveB5BD4IASuite - extends MergeIntoStructEvolutionNullnessMultiClauseTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoStructEvolutionNullnessMultiClauseSQLNameBasedColMapIdModePreserveNullSourceOnPreserveIO32QAASuite - extends MergeIntoStructEvolutionNullnessMultiClauseTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoStructEvolutionNullnessMultiClauseSQLNameBasedColMapNameModePreserveNullSourceOffPrese2A7DLQISuite - extends MergeIntoStructEvolutionNullnessMultiClauseTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoStructEvolutionNullnessMultiClauseSQLNameBasedColMapNameModePreserveNullSourceOffPreseY5PJPFYSuite - extends MergeIntoStructEvolutionNullnessMultiClauseTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoStructEvolutionNullnessMultiClauseSQLNameBasedColMapNameModePreserveNullSourceOnPreser7ZAUKFYSuite - extends MergeIntoStructEvolutionNullnessMultiClauseTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoStructEvolutionNullnessMultiClauseSQLNameBasedColMapNameModePreserveNullSourceOnPreserRU3Q5DYSuite - extends MergeIntoStructEvolutionNullnessMultiClauseTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoStructEvolutionNullnessMultiClauseSQLNameBasedPreserveNullSourceOffPreserveNullSourceUGATQTNQSuite - extends MergeIntoStructEvolutionNullnessMultiClauseTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - -class MergeIntoStructEvolutionNullnessMultiClauseSQLNameBasedPreserveNullSourceOffPreserveNullSourceUWC7MIPQSuite - extends MergeIntoStructEvolutionNullnessMultiClauseTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - -class MergeIntoStructEvolutionNullnessMultiClauseSQLNameBasedPreserveNullSourceOnPreserveNullSourceUpB5L64TISuite - extends MergeIntoStructEvolutionNullnessMultiClauseTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - -class MergeIntoStructEvolutionNullnessMultiClauseSQLNameBasedPreserveNullSourceOnPreserveNullSourceUpUYKAD7YSuite - extends MergeIntoStructEvolutionNullnessMultiClauseTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoTopLevelArrayStructEvolutionNullnessTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoTopLevelArrayStructEvolutionNullnessTests.scala index a8973377638..a26a2c966ac 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoTopLevelArrayStructEvolutionNullnessTests.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoTopLevelArrayStructEvolutionNullnessTests.scala @@ -32,102 +32,7 @@ import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ -class MergeIntoTopLevelArrayStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOffPrese53B5UFQSuite +class MergeIntoTopLevelArrayStructEvolutionNullnessSQLNameBasedSuite extends MergeIntoTopLevelArrayStructEvolutionNullnessTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelArrayStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOffPreseMJWNT3ASuite - extends MergeIntoTopLevelArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelArrayStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOnPreserSDHOQ3YSuite - extends MergeIntoTopLevelArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelArrayStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOnPreserWUUBTSQSuite - extends MergeIntoTopLevelArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelArrayStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOffPrePFWPP2ISuite - extends MergeIntoTopLevelArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelArrayStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOffPreQ33TTJISuite - extends MergeIntoTopLevelArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelArrayStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOnPres5MRQBEASuite - extends MergeIntoTopLevelArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelArrayStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOnPresT5HAGKASuite - extends MergeIntoTopLevelArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelArrayStructEvolutionNullnessSQLNameBasedPreserveNullSourceOffPreserveNullSourcNGOSKFYSuite - extends MergeIntoTopLevelArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - -class MergeIntoTopLevelArrayStructEvolutionNullnessSQLNameBasedPreserveNullSourceOffPreserveNullSourcVCDDUUISuite - extends MergeIntoTopLevelArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - -class MergeIntoTopLevelArrayStructEvolutionNullnessSQLNameBasedPreserveNullSourceOnPreserveNullSource6R2VGSQSuite - extends MergeIntoTopLevelArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - -class MergeIntoTopLevelArrayStructEvolutionNullnessSQLNameBasedPreserveNullSourceOnPreserveNullSourceG5G5COQSuite - extends MergeIntoTopLevelArrayStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoTopLevelMapStructEvolutionNullnessTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoTopLevelMapStructEvolutionNullnessTests.scala index 73dca4593a5..f6eddd146a0 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoTopLevelMapStructEvolutionNullnessTests.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoTopLevelMapStructEvolutionNullnessTests.scala @@ -32,102 +32,7 @@ import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ -class MergeIntoTopLevelMapStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOffPreservIVRBZBASuite +class MergeIntoTopLevelMapStructEvolutionNullnessSQLNameBasedSuite extends MergeIntoTopLevelMapStructEvolutionNullnessTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelMapStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOffPreservKAQJEBYSuite - extends MergeIntoTopLevelMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelMapStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOnPreserve2CQMFJASuite - extends MergeIntoTopLevelMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelMapStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOnPreserveLRUOIYQSuite - extends MergeIntoTopLevelMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelMapStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOffPrese7ZKRFZQSuite - extends MergeIntoTopLevelMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelMapStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOffPreseBJ226TYSuite - extends MergeIntoTopLevelMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelMapStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOnPreser5E2PNOASuite - extends MergeIntoTopLevelMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelMapStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOnPreser6UFQNPQSuite - extends MergeIntoTopLevelMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelMapStructEvolutionNullnessSQLNameBasedPreserveNullSourceOffPreserveNullSourceUMKC5U2ASuite - extends MergeIntoTopLevelMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - -class MergeIntoTopLevelMapStructEvolutionNullnessSQLNameBasedPreserveNullSourceOffPreserveNullSourceUPRQCVZISuite - extends MergeIntoTopLevelMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - -class MergeIntoTopLevelMapStructEvolutionNullnessSQLNameBasedPreserveNullSourceOnPreserveNullSourceUp7WIQ4DISuite - extends MergeIntoTopLevelMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - -class MergeIntoTopLevelMapStructEvolutionNullnessSQLNameBasedPreserveNullSourceOnPreserveNullSourceUpGZV4B5YSuite - extends MergeIntoTopLevelMapStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoTopLevelStructEvolutionNullnessTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoTopLevelStructEvolutionNullnessTests.scala index c05b3f5a6a2..1ce93ba19d5 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoTopLevelStructEvolutionNullnessTests.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/generatedsuites/MergeSuitesMergeIntoTopLevelStructEvolutionNullnessTests.scala @@ -32,102 +32,7 @@ import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.cdc._ import org.apache.spark.sql.delta.rowid._ -class MergeIntoTopLevelStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOffPreserveNuHUY7GZQSuite +class MergeIntoTopLevelStructEvolutionNullnessSQLNameBasedSuite extends MergeIntoTopLevelStructEvolutionNullnessTests with MergeIntoSQLMixin with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOffPreserveNuP2BPJAASuite - extends MergeIntoTopLevelStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOnPreserveNul4SO3XDASuite - extends MergeIntoTopLevelStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelStructEvolutionNullnessSQLNameBasedColMapIdModePreserveNullSourceOnPreserveNulTHLQ4DASuite - extends MergeIntoTopLevelStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableIdMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOffPreserveJI3LTLQSuite - extends MergeIntoTopLevelStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOffPreserveQXK26DISuite - extends MergeIntoTopLevelStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOnPreserveN3E2WHXISuite - extends MergeIntoTopLevelStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelStructEvolutionNullnessSQLNameBasedColMapNameModePreserveNullSourceOnPreserveNK7XS44YSuite - extends MergeIntoTopLevelStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with DeltaColumnMappingEnableNameMode - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - with MergeIntoSQLColumnMappingOverrides - -class MergeIntoTopLevelStructEvolutionNullnessSQLNameBasedPreserveNullSourceOffPreserveNullSourceUpda565RLXQSuite - extends MergeIntoTopLevelStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled - -class MergeIntoTopLevelStructEvolutionNullnessSQLNameBasedPreserveNullSourceOffPreserveNullSourceUpdaFOJBZMYSuite - extends MergeIntoTopLevelStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceDisabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - -class MergeIntoTopLevelStructEvolutionNullnessSQLNameBasedPreserveNullSourceOnPreserveNullSourceUpdatAY5MSVASuite - extends MergeIntoTopLevelStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarDisabled - -class MergeIntoTopLevelStructEvolutionNullnessSQLNameBasedPreserveNullSourceOnPreserveNullSourceUpdateStarOnSuite - extends MergeIntoTopLevelStructEvolutionNullnessTests - with MergeIntoSQLMixin - with DeltaDMLTestUtilsNameBased - with StructEvolutionPreserveNullSourceEnabled - with StructEvolutionPreserveNullSourceUpdateStarEnabled diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/hooks/metrics/UpdateMetricsHookSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/hooks/metrics/UpdateMetricsHookSuite.scala new file mode 100644 index 00000000000..229805603ae --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/hooks/metrics/UpdateMetricsHookSuite.scala @@ -0,0 +1,459 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.hooks.metrics + +import java.io.{BufferedReader, InputStreamReader, PrintWriter} +import java.net.{ServerSocket, Socket} +import java.util.concurrent.atomic.AtomicInteger + +import io.delta.storage.commit.uccommitcoordinator.UCCommitCoordinatorClient + +import org.apache.spark.sql.delta.{CommittedTransaction, DeltaLog} +import org.apache.spark.sql.delta.test.DeltaSQLCommandTest +import org.apache.spark.sql.delta.util.{CatalogTableUtils, JsonUtils} + +import org.apache.spark.SparkConf +import org.apache.spark.sql.QueryTest +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType} +import org.apache.spark.sql.test.SharedSparkSession +import org.apache.spark.sql.types.StructType + +class UpdateMetricsHookSuite extends QueryTest + with SharedSparkSession + with DeltaSQLCommandTest { + private val TEST_CATALOG_NAME = "test_catalog" + private val TEST_UC_TABLE_ID = "uc-table-id-abc" + + private def ucStorageFormat(ucTableId: String = TEST_UC_TABLE_ID): CatalogStorageFormat = { + CatalogStorageFormat( + locationUri = None, + inputFormat = None, + outputFormat = None, + serde = None, + compressed = false, + properties = Map( + UCCommitCoordinatorClient.UC_TABLE_ID_KEY -> ucTableId, + "delta.feature.catalogManaged" -> "supported" + ) + ) + } + + override protected def sparkConf: SparkConf = { + super.sparkConf + .set("spark.databricks.delta.properties.defaults.enableChangeDataFeed", "false") + } + + test("CatalogTableUtils.isUnityCatalogManagedTable: detection cases") { + val ucManaged = CatalogTable( + identifier = TableIdentifier("t", Some("default"), Some("uc_catalog")), + tableType = CatalogTableType.MANAGED, + storage = ucStorageFormat(), + schema = new StructType() + ) + assert(CatalogTableUtils.isUnityCatalogManagedTable(ucManaged), + "UC-managed table with correct storage properties should be detected") + + val noUCTableId = CatalogTable( + identifier = TableIdentifier("t", Some("default"), Some("uc_catalog")), + tableType = CatalogTableType.MANAGED, + storage = CatalogStorageFormat( + locationUri = None, + inputFormat = None, + outputFormat = None, + serde = None, + compressed = false, + properties = Map("delta.feature.catalogManaged" -> "supported") + ), + schema = new StructType() + ) + assert(!CatalogTableUtils.isUnityCatalogManagedTable(noUCTableId), + "table without UC table ID should not be detected") + + val noCatalogFeature = CatalogTable( + identifier = TableIdentifier("t", Some("default"), Some("uc_catalog")), + tableType = CatalogTableType.MANAGED, + storage = CatalogStorageFormat( + locationUri = None, + inputFormat = None, + outputFormat = None, + serde = None, + compressed = false, + properties = Map( + UCCommitCoordinatorClient.UC_TABLE_ID_KEY -> "some-id" + ) + ), + schema = new StructType() + ) + assert(!CatalogTableUtils.isUnityCatalogManagedTable(noCatalogFeature), + "table with UC ID but no catalog feature flag should not be detected") + + val emptyStorage = CatalogTable( + identifier = TableIdentifier("t", Some("default"), Some("uc_catalog")), + tableType = CatalogTableType.MANAGED, + storage = CatalogStorageFormat.empty, + schema = new StructType() + ) + assert(!CatalogTableUtils.isUnityCatalogManagedTable(emptyStorage), + "table with empty storage properties should not be detected") + } + + test("sendMetrics: sends payload with Authorization header") { + val mockServer = new SimpleMockServer(0) + try { + mockServer.setResponseCode(200) + mockServer.start() + + spark.conf.set( + s"spark.sql.catalog.$TEST_CATALOG_NAME", + "io.unitycatalog.spark.UCSingleCatalog") + spark.conf.set( + s"spark.sql.catalog.$TEST_CATALOG_NAME.uri", + s"http://localhost:${mockServer.getPort()}") + spark.conf.set( + s"spark.sql.catalog.$TEST_CATALOG_NAME.token", + "test-token-123") + + val request = ReportDeltaMetricsRequest( + tableId = "abc-123", + report = CommitReportEnvelope(CommitReport()) + ) + UpdateMetricsHook.sendMetrics( + spark, request, catalogName = Some(TEST_CATALOG_NAME)) + + assert(mockServer.getRequestCount() == 1, "Expected 1 HTTP request") + + val authHeader = mockServer.getLastHeaders().get("Authorization") + assert(authHeader.isDefined, "Authorization header should be present") + assert(authHeader.get == "Bearer test-token-123", + "Auth token must match") + + val body = mockServer.getLastRequestBody() + val actualPayload = + JsonUtils.fromJson[Map[String, Any]](body) + val expectedPayload = + JsonUtils.fromJson[Map[String, Any]](JsonUtils.toJson(request)) + assert(actualPayload == expectedPayload, + "request JSON should match expected payload") + } finally { + mockServer.stop() + } + } + + test("sendMetrics: auth.type=static with auth.token") { + val authStaticCatalog = "auth_static_catalog" + val mockServer = new SimpleMockServer(0) + try { + mockServer.setResponseCode(200) + mockServer.start() + + spark.conf.set( + s"spark.sql.catalog.$authStaticCatalog", + "io.unitycatalog.spark.UCSingleCatalog") + spark.conf.set( + s"spark.sql.catalog.$authStaticCatalog.uri", + s"http://localhost:${mockServer.getPort()}") + spark.conf.set( + s"spark.sql.catalog.$authStaticCatalog.auth.type", + "static") + spark.conf.set( + s"spark.sql.catalog.$authStaticCatalog.auth.token", + "auth-static-token-456") + + val request = ReportDeltaMetricsRequest( + tableId = "auth-test-id", + report = CommitReportEnvelope(CommitReport()) + ) + UpdateMetricsHook.sendMetrics( + spark, request, catalogName = Some(authStaticCatalog)) + + assert(mockServer.getRequestCount() == 1, "Expected 1 HTTP request") + val authHeader = mockServer.getLastHeaders().get("Authorization") + assert(authHeader.isDefined, + "Authorization header should be present") + assert(authHeader.get == "Bearer auth-static-token-456", + "Auth token from auth.token must match") + } finally { + mockServer.stop() + } + } + + test("run(): fires HTTP POST for UC-managed table") { + val mockServer = new SimpleMockServer(0) + try { + mockServer.setResponseCode(200) + mockServer.start() + + withTempDir { dir => + spark.range(10).write.format("delta").save(dir.getCanonicalPath) + val deltaLog = DeltaLog.forTable(spark, dir.getCanonicalPath) + val snapshot = deltaLog.snapshot + + spark.conf.set( + "spark.sql.catalog.spark_catalog", + "io.unitycatalog.spark.UCSingleCatalog") + spark.conf.set( + "spark.sql.catalog.spark_catalog.uri", + s"http://localhost:${mockServer.getPort()}") + spark.conf.set( + "spark.sql.catalog.spark_catalog.token", "smoke-token") + + val catalogTable = CatalogTable( + identifier = TableIdentifier( + "t", Some("default"), Some("spark_catalog")), + tableType = CatalogTableType.MANAGED, + storage = ucStorageFormat(), + schema = new StructType() + ) + + val txn = CommittedTransaction( + txnId = "smoke-txn", + deltaLog = deltaLog, + catalogTable = Some(catalogTable), + readSnapshot = snapshot, + committedVersion = 0L, + committedActions = Seq.empty, + postCommitSnapshot = snapshot, + postCommitHooks = Seq.empty, + txnExecutionTimeMs = 0L, + needsCheckpoint = false, + partitionsAddedToOpt = None, + isBlindAppend = true + ) + + UpdateMetricsHook(Some(catalogTable)).run(spark, txn) + + assert(mockServer.getRequestCount() == 1, + "Expected exactly 1 HTTP POST") + val body = mockServer.getLastRequestBody() + val expectedRequest = UpdateMetricsHook.buildRequest( + TEST_UC_TABLE_ID, Seq.empty, 0L) + val actualPayload = + JsonUtils.fromJson[Map[String, Any]](body) + val expectedPayload = + JsonUtils.fromJson[Map[String, Any]]( + JsonUtils.toJson(expectedRequest)) + assert(actualPayload == expectedPayload, + "smoke test payload should match expected JSON") + } + } finally { + mockServer.stop() + } + } + + test("run(): hook does not crash on HTTP 5xx (best-effort)") { + val mockServer = new SimpleMockServer(0) + try { + mockServer.setResponseCode(500) + mockServer.start() + + withTempDir { dir => + spark.range(10).write.format("delta") + .save(dir.getCanonicalPath) + val deltaLog = DeltaLog.forTable( + spark, dir.getCanonicalPath) + val snapshot = deltaLog.snapshot + + spark.conf.set( + "spark.sql.catalog.spark_catalog", + "io.unitycatalog.spark.UCSingleCatalog") + spark.conf.set( + "spark.sql.catalog.spark_catalog.uri", + s"http://localhost:${mockServer.getPort()}") + spark.conf.set( + "spark.sql.catalog.spark_catalog.token", + "error-token") + + val catalogTable = CatalogTable( + identifier = TableIdentifier( + "t", Some("default"), Some("spark_catalog")), + tableType = CatalogTableType.MANAGED, + storage = ucStorageFormat(), + schema = new StructType() + ) + + val txn = CommittedTransaction( + txnId = "error-txn", + deltaLog = deltaLog, + catalogTable = Some(catalogTable), + readSnapshot = snapshot, + committedVersion = 0L, + committedActions = Seq.empty, + postCommitSnapshot = snapshot, + postCommitHooks = Seq.empty, + txnExecutionTimeMs = 0L, + needsCheckpoint = false, + partitionsAddedToOpt = None, + isBlindAppend = true + ) + + UpdateMetricsHook(Some(catalogTable)).run(spark, txn) + + assert(mockServer.getRequestCount() == 1, + "request should still be sent on 5xx") + } + } finally { + mockServer.stop() + } + } + + test("run(): skips metrics when table ID not in storage properties") { + withTempDir { dir => + spark.range(1).write.format("delta").save(dir.getCanonicalPath) + val deltaLog = DeltaLog.forTable(spark, dir.getCanonicalPath) + val snapshot = deltaLog.snapshot + + val catalogTable = CatalogTable( + identifier = TableIdentifier( + "t", Some("default"), Some("spark_catalog")), + tableType = CatalogTableType.MANAGED, + storage = CatalogStorageFormat( + locationUri = None, + inputFormat = None, + outputFormat = None, + serde = None, + compressed = false, + properties = Map( + "delta.feature.catalogManaged" -> "supported" + ) + ), + schema = new StructType() + ) + + val txn = CommittedTransaction( + txnId = "skip-txn", + deltaLog = deltaLog, + catalogTable = Some(catalogTable), + readSnapshot = snapshot, + committedVersion = 0L, + committedActions = Seq.empty, + postCommitSnapshot = snapshot, + postCommitHooks = Seq.empty, + txnExecutionTimeMs = 0L, + needsCheckpoint = false, + partitionsAddedToOpt = None, + isBlindAppend = true + ) + + UpdateMetricsHook(Some(catalogTable)).run(spark, txn) + } + } +} + +class SimpleMockServer(port: Int) { + private var serverSocket: ServerSocket = _ + private var serverThread: Thread = _ + @volatile private var running = false + private var responseCode = 200 + private val requestCount = new AtomicInteger(0) + @volatile private var lastRequestBody = "" + @volatile private var lastHeaders = Map[String, String]() + private var actualPort = port + + def setResponseCode(code: Int): Unit = { responseCode = code } + + def getPort(): Int = actualPort + + def getRequestCount(): Int = requestCount.get() + + def getLastRequestBody(): String = lastRequestBody + + def getLastHeaders(): Map[String, String] = lastHeaders + + def start(): Unit = { + serverSocket = new ServerSocket(port) + actualPort = serverSocket.getLocalPort + running = true + + serverThread = new Thread(new Runnable { + override def run(): Unit = { + while (running) { + try { + val clientSocket = serverSocket.accept() + handleRequest(clientSocket) + } catch { + case _: java.net.SocketException if !running => + case _: Exception => + } + } + } + }) + serverThread.setDaemon(true) + serverThread.start() + } + + private def handleRequest(clientSocket: Socket): Unit = { + try { + val in = new BufferedReader( + new InputStreamReader(clientSocket.getInputStream)) + val out = new PrintWriter(clientSocket.getOutputStream, true) + + in.readLine() + + val headers = scala.collection.mutable.Map[String, String]() + var contentLength = 0 + var line = in.readLine() + while (line != null && line.nonEmpty) { + val parts = line.split(":", 2) + if (parts.length == 2) { + val key = parts(0).trim + val value = parts(1).trim + headers(key) = value + if (key.equalsIgnoreCase("Content-Length")) { + contentLength = value.toInt + } + } + line = in.readLine() + } + + val body = new Array[Char](contentLength) + if (contentLength > 0) in.read(body, 0, contentLength) + + requestCount.incrementAndGet() + lastRequestBody = new String(body) + lastHeaders = headers.toMap + + out.write( + s"HTTP/1.1 $responseCode ${statusMessage(responseCode)}\r\n") + out.write("Content-Type: application/json\r\n") + out.write("Content-Length: 2\r\n") + out.write("\r\n") + out.write("{}") + out.flush() + + clientSocket.close() + } catch { + case _: java.io.IOException => + } + } + + private def statusMessage(code: Int): String = code match { + case 200 => "OK" + case 400 => "Bad Request" + case 500 => "Internal Server Error" + case _ => "Unknown" + } + + def stop(): Unit = { + running = false + if (serverSocket != null && !serverSocket.isClosed) serverSocket.close() + if (serverThread != null) { + serverThread.interrupt() + serverThread.join(1000) + } + } +} diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlannedTableSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlannedTableSuite.scala index 771c7a148af..231ea7493ed 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlannedTableSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlannedTableSuite.scala @@ -265,7 +265,7 @@ class ServerSidePlannedTableSuite extends QueryTest with DeltaSQLCommandTest { // Verify the metadata has expected defaults assert(metadata.catalogName == "my_catalog") assert(metadata.planningEndpointUri == "") - assert(metadata.authToken.isEmpty) + assert(metadata.tokenSupplier.isEmpty) assert(metadata.tableProperties.isEmpty) } @@ -274,7 +274,7 @@ class ServerSidePlannedTableSuite extends QueryTest with DeltaSQLCommandTest { val metadata = UnityCatalogMetadata( catalogName = "test_catalog", ucUri = ucUri, - ucToken = "test-token", + tokenSupplier = Some(() => "test-token"), tableProps = Map.empty ) @@ -287,6 +287,76 @@ class ServerSidePlannedTableSuite extends QueryTest with DeltaSQLCommandTest { assert(metadata.planningEndpointUri == expectedEndpoint) } + test("extractAuthConfig: OAuth config") { + val cat = "oauthCat" + withSQLConf( + s"spark.sql.catalog.$cat.auth.type" -> "oauth", + s"spark.sql.catalog.$cat.auth.oauth.uri" -> + "https://oauth.example.com/token", + s"spark.sql.catalog.$cat.auth.oauth.clientId" -> + "client123", + s"spark.sql.catalog.$cat.auth.oauth.clientSecret" -> + "secret456" + ) { + val config = UnityCatalogMetadata + .extractAuthConfig(spark, cat) + assert(config("type") == "oauth") + assert(config("oauth.uri") == + "https://oauth.example.com/token") + assert(config("oauth.clientId") == "client123") + assert(config.size == 4) + } + } + + test("extractAuthConfig: legacy token as static") { + val cat = "legacyCat" + withSQLConf( + s"spark.sql.catalog.$cat.token" -> "dapi-token" + ) { + val config = UnityCatalogMetadata + .extractAuthConfig(spark, cat) + assert(config("type") == "static") + assert(config("token") == "dapi-token") + assert(config.size == 2) + } + } + + test("extractAuthConfig: error if both token formats") { + val cat = "conflictCat" + withSQLConf( + s"spark.sql.catalog.$cat.auth.token" -> "new", + s"spark.sql.catalog.$cat.token" -> "old" + ) { + val e = intercept[IllegalArgumentException] { + UnityCatalogMetadata.extractAuthConfig(spark, cat) + } + assert(e.getMessage.contains("configured twice")) + } + } + + test("extractAuthConfig: legacy token wins over auth.*") { + val cat = "legacyWinsCat" + withSQLConf( + s"spark.sql.catalog.$cat.auth.type" -> "oauth", + s"spark.sql.catalog.$cat.auth.oauth.uri" -> + "https://oauth.example.com", + s"spark.sql.catalog.$cat.token" -> "dapi-legacy" + ) { + val config = UnityCatalogMetadata + .extractAuthConfig(spark, cat) + // Legacy .token always wins (converted to static) + assert(config("type") == "static") + assert(config("token") == "dapi-legacy") + assert(config.size == 2) + } + } + + test("extractAuthConfig: empty when no auth") { + val config = UnityCatalogMetadata + .extractAuthConfig(spark, "nonExistentCat") + assert(config.isEmpty) + } + test("simple EqualTo filter pushed to planning client") { withPushdownCapturingEnabled { sql("SELECT id, name, value FROM test_db.shared_test WHERE id = 2").collect() @@ -485,4 +555,36 @@ class ServerSidePlannedTableSuite extends QueryTest with DeltaSQLCommandTest { assert(capturedProjection.isEmpty, "Should not fire a planTable request for EXPLAIN") } } + + test("ServerSidePlannedTable closes planning client on close") { + var clientClosed = false + val trackingClient = new ServerSidePlanningClient { + override def planScan( + databaseName: String, + table: String, + filterOption: Option[Filter], + projectionOption: Option[Seq[String]], + limitOption: Option[Int]): ScanPlan = ScanPlan(Seq.empty) + override def canConvertFilters(filters: Array[Filter]): Boolean = true + override def close(): Unit = { clientClosed = true } + } + + val table = new ServerSidePlannedTable( + spark, "test_db", "test_table", new org.apache.spark.sql.types.StructType(), trackingClient) + assert(!clientClosed, "Client should not be closed before table.close()") + table.close() + assert(clientClosed, "Client should be closed after table.close()") + } + + test("planning client is closed after scan completes") { + withPushdownCapturingEnabled { + assert(!TestServerSidePlanningClient.isClientClosed, + "Client should not be closed before query execution") + + sql("SELECT id FROM test_db.shared_test").collect() + + assert(TestServerSidePlanningClient.isClientClosed, + "Planning client should be closed after scan plan is obtained") + } + } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/TestServerSidePlanningClient.scala b/spark/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/TestServerSidePlanningClient.scala index 21ce5501ed1..11b149fd18d 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/TestServerSidePlanningClient.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/serverSidePlanning/TestServerSidePlanningClient.scala @@ -94,6 +94,10 @@ class TestServerSidePlanningClient(spark: SparkSession) extends ServerSidePlanni TestServerSidePlanningClient.filtersConvertible } + override def close(): Unit = { + TestServerSidePlanningClient.clientClosed = true + } + private def getFileFormat(path: Path): String = "parquet" } @@ -106,10 +110,12 @@ object TestServerSidePlanningClient { private var capturedProjection: Option[Seq[String]] = None private var capturedLimit: Option[Int] = None private var filtersConvertible: Boolean = true // Default: all filters convertible + private[serverSidePlanning] var clientClosed: Boolean = false def getCapturedFilter: Option[Filter] = capturedFilter def getCapturedProjection: Option[Seq[String]] = capturedProjection def getCapturedLimit: Option[Int] = capturedLimit + def isClientClosed: Boolean = clientClosed /** * Configure whether filters should be treated as convertible. @@ -124,6 +130,7 @@ object TestServerSidePlanningClient { capturedProjection = None capturedLimit = None filtersConvertible = true // Reset to default + clientClosed = false } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/stats/DataSkippingDeltaTests.scala b/spark/src/test/scala/org/apache/spark/sql/delta/stats/DataSkippingDeltaTests.scala index 0c6f961a1ae..ed76da9a302 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/stats/DataSkippingDeltaTests.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/stats/DataSkippingDeltaTests.scala @@ -1671,41 +1671,92 @@ trait DataSkippingDeltaTestsBase extends QueryTest s"STRUCT(CAST(strTs AS $timestampType) AS ts) AS nested") val tempDir = Utils.createTempDir() - val r = DeltaLog.forTable(spark, tempDir) - df.coalesce(1).write.format("delta").save(r.dataPath.toString) + val log = DeltaLog.forTable(spark, tempDir) + df.coalesce(1).write.format("delta").save(log.dataPath.toString) + + checkSkipping( + log, + // Check to ensure that the value actually in the file is always in range queries. + hits = Seq( + s"""ts >= cast("2019-09-09 01:02:03.456789" AS $timestampType)""", + s"""ts <= cast("2019-09-09 01:02:03.456789" AS $timestampType)""", + s"""nested.ts >= cast("2019-09-09 01:02:03.456789" AS $timestampType)""", + s"""nested.ts <= cast("2019-09-09 01:02:03.456789" AS $timestampType)""", + s"""TS >= cast("2019-09-09 01:02:03.456789" AS $timestampType)""", + s"""nEstED.tS >= cast("2019-09-09 01:02:03.456789" AS $timestampType)"""), + // Check the range of values that are far enough away to be data skipped. Note that the + // values are aligned with millisecond boundaries because of the JSON serialization + // truncation. + misses = Seq( + s"""ts >= cast("2019-09-09 01:02:03.457001" AS $timestampType)""", + s"""ts <= cast("2019-09-04 01:02:03.455999" AS $timestampType)""", + s"""nested.ts >= cast("2019-09-09 01:02:03.457001" AS $timestampType)""", + s"""nested.ts <= cast("2019-09-09 01:02:03.455999" AS $timestampType)""", + s"""TS >= cast("2019-09-09 01:02:03.457001" AS $timestampType)""", + s"""nEstED.tS >= cast("2019-09-09 01:02:03.457001" AS $timestampType)"""), + data = data, + checkEmptyUnusedFiltersForHits = false) + } + } - // Check to ensure that the value actually in the file is always in range queries. - val hits = Seq( - s"""ts >= cast("2019-09-09 01:02:03.456789" AS $timestampType)""", - s"""ts <= cast("2019-09-09 01:02:03.456789" AS $timestampType)""", - s"""nested.ts >= cast("2019-09-09 01:02:03.456789" AS $timestampType)""", - s"""nested.ts <= cast("2019-09-09 01:02:03.456789" AS $timestampType)""", - s"""TS >= cast("2019-09-09 01:02:03.456789" AS $timestampType)""", - s"""nEstED.tS >= cast("2019-09-09 01:02:03.456789" AS $timestampType)""") - - // Check the range of values that are far enough away to be data skipped. Note that the values - // are aligned with millisecond boundaries because of the JSON serialization truncation. - val misses = Seq( - s"""ts >= cast("2019-09-09 01:02:03.457001" AS $timestampType)""", - s"""ts <= cast("2019-09-04 01:02:03.455999" AS $timestampType)""", - s"""nested.ts >= cast("2019-09-09 01:02:03.457001" AS $timestampType)""", - s"""nested.ts <= cast("2019-09-09 01:02:03.455999" AS $timestampType)""", - s"""TS >= cast("2019-09-09 01:02:03.457001" AS $timestampType)""", - s"""nEstED.tS >= cast("2019-09-09 01:02:03.457001" AS $timestampType)""") - - hits.foreach { predicate => - Given(predicate) - if (filesRead(r, predicate) != 1) { - failPretty(s"Expected hit but got miss for $predicate", predicate, data) - } - } + for (timestampType <- Seq("TIMESTAMP", "TIMESTAMP_NTZ")) { + test(s"data skipping on $timestampType with Long.MaxValue") { + val maxVal = "294247-01-10 04:00:54.775807Z" + val tempDir = Utils.createTempDir() + val log = DeltaLog.forTable(spark, tempDir) + Seq(maxVal).toDF("strTs") + .selectExpr(s"CAST(strTs AS $timestampType) AS ts") + .coalesce(1) + .write + .format("delta") + .save(log.dataPath.toString) + + checkSkipping( + log, + hits = Seq( + s"""ts >= cast("$maxVal" AS $timestampType)""", + s"""ts >= "$maxVal"""", + s"""ts >= cast("2019-09-09 01:02:03.457001" AS $timestampType)""", + // This still hits because of JSON truncation to milliseconds + s"""ts < cast("$maxVal" AS $timestampType)""".stripMargin), + misses = Seq( + s"""ts <= cast("2019-09-09 01:02:03.457001" AS $timestampType)""", + s"""ts > cast("$maxVal" AS $timestampType)"""), + data = maxVal, + checkEmptyUnusedFiltersForHits = false) + } + } - misses.foreach { predicate => - Given(predicate) - if (filesRead(r, predicate) != 0) { - failPretty(s"Expected miss but got hit for $predicate", predicate, data) - } - } + for (timestampType <- Seq("TIMESTAMP", "TIMESTAMP_NTZ")) { + test(s"data skipping on $timestampType near Long.MaxValue") { + val tempDir = Utils.createTempDir() + val log = DeltaLog.forTable(spark, tempDir) + + val nearMaxMicros = Long.MaxValue - 999L + + // Create DataFrame with the near-max timestamp value + Seq(nearMaxMicros).toDF("microsSinceEpoch") + .selectExpr(s"TIMESTAMP_MICROS(microsSinceEpoch) AS ts") + .selectExpr(s"CAST(ts AS $timestampType) AS ts") + .coalesce(1) + .write + .format("delta") + .save(log.dataPath.toString) + + checkSkipping( + log, + // maxValue of the stats on ts will be saturated to Long.MaxValue instead + // of being added 1000 microseconds, which will cause a long overflow. + hits = Seq( + s"ts >= TIMESTAMP_MICROS($nearMaxMicros)", + s"ts >= TIMESTAMP_MICROS(${nearMaxMicros - 100})", + s"""ts >= cast("2019-09-09 01:02:03.457001" AS $timestampType)""", + s"ts >= TIMESTAMP_MICROS(${Long.MaxValue - 1000})", + s"ts < TIMESTAMP_MICROS($nearMaxMicros)"), + misses = Seq( + s"""ts <= cast("2019-09-09 01:02:03.457001" AS $timestampType)"""), + data = nearMaxMicros.toString, + checkEmptyUnusedFiltersForHits = false) } } diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/stats/PartitionLikeDataSkippingSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/stats/PartitionLikeDataSkippingSuite.scala index 2dd3d595694..ce0d83159e7 100644 --- a/spark/src/test/scala/org/apache/spark/sql/delta/stats/PartitionLikeDataSkippingSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/delta/stats/PartitionLikeDataSkippingSuite.scala @@ -71,7 +71,7 @@ trait PartitionLikeDataSkippingSuiteBase minNumFilesToApply.toString) { // Execute the query with partition-like filters and validate that the result matches. val res = sql(query).collect() - assert(res.sameElements(baseResult)) + assert(res.sortBy(_.toString).sameElements(baseResult.sortBy(_.toString))) val predicates = sql(query).queryExecution.optimizedPlan.collect { case Filter(condition, _) => condition diff --git a/spark/src/test/scala/org/apache/spark/sql/delta/stats/PreparedDeltaFileIndexRowCountSuite.scala b/spark/src/test/scala/org/apache/spark/sql/delta/stats/PreparedDeltaFileIndexRowCountSuite.scala new file mode 100644 index 00000000000..09cf3dff5f8 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/delta/stats/PreparedDeltaFileIndexRowCountSuite.scala @@ -0,0 +1,150 @@ +/* + * Copyright (2021) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.delta.stats + +import org.apache.spark.sql.delta.{DeltaLog, DeltaTable} +import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.delta.test.DeltaSQLCommandTest + +import org.apache.spark.sql.{DataFrame, QueryTest} +import org.apache.spark.sql.functions._ +import org.apache.spark.sql.test.SharedSparkSession + +/** + * Test suite to verify when preparedScan.scanned.rows is populated in PreparedDeltaFileIndex, + * and the behavior of the DELTA_ALWAYS_COLLECT_STATS flag. + */ +class PreparedDeltaFileIndexRowCountSuite + extends QueryTest + with DeltaSQLCommandTest { + + import testImplicits._ + + private def getDeltaScan(df: DataFrame): DeltaScan = { + val scans = df.queryExecution.optimizedPlan.collect { + case DeltaTable(prepared: PreparedDeltaFileIndex) => prepared.preparedScan + } + assert(scans.size == 1, s"Expected 1 DeltaScan, found ${scans.size}") + scans.head + } + + /** + * Test utility that creates a partitioned Delta table and verifies scanned.rows and + * scanned.logicalRows behavior. + * + * @param alwaysCollectStats value of the DELTA_ALWAYS_COLLECT_STATS flag + * @param queryTransform function to transform the base DataFrame (apply filters) + * @param expectedRowsDefined whether scanned.rows should be defined + * @param expectedRowCount expected row count if defined (None to skip validation) + * @param expectedLogicalRowsDefined whether scanned.logicalRows should be defined + * (defaults to same as expectedRowsDefined) + * @param expectedLogicalRowCount expected logical row count if defined (None to skip validation) + */ + private def testRowCountBehavior( + alwaysCollectStats: Boolean, + queryTransform: DataFrame => DataFrame, + expectedRowsDefined: Boolean, + expectedRowCount: Option[Long] = None, + expectedLogicalRowsDefined: Option[Boolean] = None, + expectedLogicalRowCount: Option[Long] = None): Unit = { + withTempDir { dir => + withSQLConf(DeltaSQLConf.DELTA_COLLECT_STATS.key -> "true") { + spark.range(100).toDF("id") + .withColumn("part", $"id" % 4) + .repartition(4) + .write.format("delta").partitionBy("part").save(dir.getAbsolutePath) + } + + DeltaLog.clearCache() + + withSQLConf(DeltaSQLConf.DELTA_ALWAYS_COLLECT_STATS.key -> alwaysCollectStats.toString) { + val df = spark.read.format("delta").load(dir.getAbsolutePath) + val scan = getDeltaScan(queryTransform(df)) + + if (expectedRowsDefined) { + assert(scan.scanned.rows.isDefined, "scanned.rows should be defined") + expectedRowCount.foreach { expected => + assert(scan.scanned.rows.get == expected, + s"Expected $expected rows, got ${scan.scanned.rows.get}") + } + } else { + assert(scan.scanned.rows.isEmpty, "scanned.rows should be None") + } + + // logicalRows should follow the same defined/undefined pattern as rows by default + val logicalDefined = expectedLogicalRowsDefined.getOrElse(expectedRowsDefined) + if (logicalDefined) { + assert(scan.scanned.logicalRows.isDefined, "scanned.logicalRows should be defined") + expectedLogicalRowCount.foreach { expected => + assert(scan.scanned.logicalRows.get == expected, + s"Expected $expected logical rows, got ${scan.scanned.logicalRows.get}") + } + } else { + assert(scan.scanned.logicalRows.isEmpty, "scanned.logicalRows should be None") + } + } + } + } + + // Define query cases: (name, transform function, always collects rows) + private val queryCases: Seq[(String, DataFrame => DataFrame, Boolean)] = Seq( + ("no filter", identity[DataFrame], false), + ("TrueLiteral filter", _.where(lit(true)), false), + ("partition filter only", _.where($"part" === 1), false), + ("data filter", _.where($"id" === 50), true), + ("partition + data filter", _.where($"part" === 1).where($"id" === 49), true) + ) + + // Grid test: all query cases x flag values + for { + (caseName, queryTransform, alwaysCollectsRows) <- queryCases + alwaysCollectStats <- Seq(false, true) + } { + val flagDesc = s"alwaysCollectStats=$alwaysCollectStats" + // If the query type always collects rows, rows is always defined; otherwise depends on flag + val expectedRowsDefined = alwaysCollectsRows || alwaysCollectStats + + test(s"$caseName - $flagDesc") { + testRowCountBehavior( + alwaysCollectStats = alwaysCollectStats, + queryTransform = queryTransform, + expectedRowsDefined = expectedRowsDefined + ) + } + } + + test("alwaysCollectStats with missing stats returns None") { + withTempDir { dir => + // Create table without stats + withSQLConf(DeltaSQLConf.DELTA_COLLECT_STATS.key -> "false") { + spark.range(100).toDF("id") + .write.format("delta").save(dir.getAbsolutePath) + } + + DeltaLog.clearCache() + + withSQLConf(DeltaSQLConf.DELTA_ALWAYS_COLLECT_STATS.key -> "true") { + val df = spark.read.format("delta").load(dir.getAbsolutePath) + val scan = getDeltaScan(df) + assert(scan.scanned.rows.isEmpty, "scanned.rows should be None when stats are missing") + assert(scan.scanned.logicalRows.isEmpty, + "scanned.logicalRows should be None when stats are missing") + } + } + } + +} diff --git a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableBlockMetadataUpdateTest.java b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableBlockMetadataUpdateTest.java new file mode 100644 index 00000000000..0ffa51e5f23 --- /dev/null +++ b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableBlockMetadataUpdateTest.java @@ -0,0 +1,377 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.sparkuctest; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.Test; + +/** + * Tests that schema-changing and property-changing operations are blocked on Unity Catalog managed + * (CatalogOwned) tables — regardless of which layer does the blocking. + * + *

    There are two distinct layers of protection: + * + *

      + *
    1. Kill switch in {@code OptimisticTransaction.updateMetadata()}: blocks any + * commit that would change schema, partitions, description, or configuration on an existing + * CatalogOwned table. A second guard in {@code prepareCommit()} and {@code commitLarge()} + * blocks {@code delta.clustering} {@code DomainMetadata} changes (e.g. via RESTORE TABLE to a + * version written by an older client that had different clustering columns). + *
    2. UC catalog layer in {@code UCSingleCatalog}: {@code alterTable()} throws + * {@code UnsupportedOperationException} for all ALTER TABLE variants. INSERT OVERWRITE with + * {@code overwriteSchema=true} and {@code CREATE OR REPLACE TABLE} both route through REPLACE + * TABLE AS SELECT (RTAS) because {@code UCSingleCatalog} does not implement {@code + * StagingTableCatalog}; RTAS is not supported in OSS Delta. + *
    + * + *

    EXTERNAL tables are not CatalogOwned and are NOT affected by the kill switch; they continue to + * allow schema evolution as before. + */ +public class UCDeltaTableBlockMetadataUpdateTest extends UCDeltaTableIntegrationBaseTest { + + // Error produced by the kill switch in OptimisticTransaction.updateMetadata(). + private static final String KILL_SWITCH_ERROR = + "Metadata changes on Unity Catalog managed tables"; + + // Error produced by the clustering kill switch in commitLarge(). + private static final String CLUSTERING_KILL_SWITCH_ERROR = + "Clustering column changes on Unity Catalog managed tables"; + + // Error produced by UCSingleCatalog.alterTable() for all ALTER TABLE variants. + private static final String ALTER_TABLE_ERROR = "Altering a table is not supported yet"; + + // Error produced by OSS Delta when REPLACE TABLE AS SELECT (RTAS) is attempted. + // Triggered by CREATE OR REPLACE TABLE and DataFrame saveAsTable(overwrite+overwriteSchema) + // when the target catalog does not implement StagingTableCatalog. + private static final String RTAS_ERROR = "REPLACE TABLE AS SELECT (RTAS) is not supported"; + + // --------------------------------------------------------------------------- + // Kill-switch tests: operations blocked by OptimisticTransaction.updateMetadata() + // --------------------------------------------------------------------------- + + /** + * INSERT with {@code autoMerge=true} and MERGE INTO with schema evolution must be blocked by the + * kill switch in {@code updateMetadata()} on CatalogOwned tables. The kill switch covers all + * metadata fields (schema, partitions, description, properties); schema evolution via autoMerge + * is the primary user-facing path that reaches it without going through ALTER TABLE. + */ + @Test + public void testMetadataChangesViaWritesAreBlocked() throws Exception { + withNewTable( + "block_schema_evolution_target", + "id INT, name STRING", + TableType.MANAGED, + targetTable -> { + sql("INSERT INTO %s VALUES (1, 'initial')", targetTable); + withNewTable( + "block_schema_evolution_source", + "id INT, name STRING, extra STRING", + TableType.EXTERNAL, + sourceTable -> { + sql("INSERT INTO %s VALUES (2, 'new', 'extra_value')", sourceTable); + sql("SET spark.databricks.delta.schema.autoMerge.enabled = true"); + try { + // INSERT with autoMerge introduces a new column. + assertThrowsWithCauseContaining( + KILL_SWITCH_ERROR, + () -> sql("INSERT INTO %s SELECT * FROM %s", targetTable, sourceTable)); + + // MERGE INTO with autoMerge introduces a new column from the source. + assertThrowsWithCauseContaining( + KILL_SWITCH_ERROR, + () -> + sql( + "MERGE INTO %s AS target " + + "USING %s AS source " + + "ON target.id = source.id " + + "WHEN NOT MATCHED THEN INSERT *", + targetTable, sourceTable)); + } finally { + sql("SET spark.databricks.delta.schema.autoMerge.enabled = false"); + } + }); + }); + } + + // --------------------------------------------------------------------------- + // UC catalog layer tests: operations blocked by UCSingleCatalog before reaching Delta + // --------------------------------------------------------------------------- + + /** + * All ALTER TABLE variants on a CatalogOwned table must be blocked by {@code + * UCSingleCatalog.alterTable()}, which throws {@code UnsupportedOperationException} for every + * table change regardless of the specific operation. + * + *

    Covered operations: SET TBLPROPERTIES (configuration change), ADD COLUMNS (schema change), + * and CLUSTER BY (clustering change). All share the same managed table and each throws before + * modifying anything. + */ + @Test + public void testAlterTableOperationsAreBlocked() throws Exception { + withNewTable( + "block_alter_table_test", + "id INT, name STRING", + TableType.MANAGED, + tableName -> { + sql("INSERT INTO %s VALUES (1, 'initial')", tableName); + + // ALTER TABLE SET TBLPROPERTIES would change configuration. + assertThrowsWithCauseContaining( + ALTER_TABLE_ERROR, + () -> sql("ALTER TABLE %s SET TBLPROPERTIES ('custom.key' = 'value')", tableName)); + + // ALTER TABLE ADD COLUMNS would change the schema. + assertThrowsWithCauseContaining( + ALTER_TABLE_ERROR, () -> sql("ALTER TABLE %s ADD COLUMNS (extra STRING)", tableName)); + + // ALTER TABLE CLUSTER BY would change clustering columns. + assertThrowsWithCauseContaining( + ALTER_TABLE_ERROR, () -> sql("ALTER TABLE %s CLUSTER BY (id)", tableName)); + }); + } + + /** + * RESTORE TABLE to a version with unchanged clustering must succeed. The clustering kill switch + * only fires when clustering actually changes. + */ + @Test + public void testRestoreTableWithUnchangedClusteringSucceeds() throws Exception { + String tableName = fullTableName("restore_unchanged_clustering_test"); + try { + sql( + "CREATE TABLE %s (id INT, name STRING) USING DELTA CLUSTER BY (id)" + + " TBLPROPERTIES ('delta.feature.catalogManaged'='supported')", + tableName); + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); + long versionAfterInsert = currentVersion(tableName); + // Restore to version 0 (before the insert): clustering is unchanged, must succeed. + sql("RESTORE TABLE %s TO VERSION AS OF %d", tableName, versionAfterInsert - 1); + check(tableName, List.of()); + } finally { + sql("DROP TABLE IF EXISTS %s", tableName); + } + } + + /** + * RESTORE TABLE to a version whose clustering differs from the current version must be blocked by + * the kill switch in {@code commitLarge()}. + * + *

    A table can have different clustering at different versions if an older client or another + * connector wrote a version before this guard was in place. This test simulates that by writing a + * Delta log entry with different clustering directly to the table's underlying storage (bypassing + * the kill switch), then attempting a RESTORE that would change clustering back. + * + *

    This test is local-only: it needs direct filesystem access to write the fake commit, which + * is only available when backed by {@code S3CredentialFileSystem} (local UC). + */ + @Test + public void testRestoreTableWithClusteringChangeIsBlocked() throws Exception { + Assumptions.assumeFalse( + isUCRemoteConfigured(), "Requires local filesystem access (local UC only)"); + + String tableName = fullTableName("restore_clustering_change_test"); + try { + sql( + "CREATE TABLE %s (id INT, name STRING) USING DELTA CLUSTER BY (id)" + + " TBLPROPERTIES ('delta.feature.catalogManaged'='supported')", + tableName); + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); + long insertVersion = currentVersion(tableName); + + // Simulate an older client writing a version with different clustering directly to the + // table's storage, bypassing the kill switch. This is how a real-world scenario could arise. + String s3Location = + sql("DESCRIBE FORMATTED %s", tableName).stream() + .filter(r -> r.size() >= 2 && "Location".equalsIgnoreCase(r.get(0).trim())) + .map(r -> r.get(1).trim()) + .findFirst() + .orElseThrow(); + // Convert URI to a local filesystem path: + // - S3CredentialFileSystem maps s3://fakeS3Bucket/abs/path → /abs/path + // - Local UC may return a file: URI directly + String localTablePath; + if (s3Location.startsWith("s3://")) { + localTablePath = s3Location.replaceAll("s3://[^/]+", ""); + } else if (s3Location.startsWith("file:")) { + localTablePath = s3Location.replaceAll("^file:/+", "/"); + } else { + localTablePath = s3Location; + } + long hackedVersion = insertVersion + 1; + Path hackedCommitFile = + Paths.get(localTablePath, "_delta_log", String.format("%020d.json", hackedVersion)); + // Write a minimal Delta commit with delta.clustering on 'name' instead of 'id'. + String clusteringOnName = "{\\\"clusteringColumns\\\":[[\\\"name\\\"]]}"; + Files.writeString( + hackedCommitFile, + "{\"commitInfo\":{\"timestamp\":1000000000000,\"inCommitTimestamp\":1000000000000," + + "\"operation\":\"MANUAL UPDATE\",\"operationParameters\":{},\"isBlindAppend\":false}}\n" + + "{\"domainMetadata\":{\"domain\":\"delta.clustering\"," + + "\"configuration\":\"" + + clusteringOnName + + "\",\"removed\":false}}\n"); + + // RESTORE to a version before the hacked commit: the current snapshot now shows 'name' + // clustering, so restoring to 'id' clustering fires the kill switch. + assertThrowsWithCauseContaining( + CLUSTERING_KILL_SWITCH_ERROR, + () -> sql("RESTORE TABLE %s TO VERSION AS OF %d", tableName, insertVersion - 1)); + } finally { + sql("DROP TABLE IF EXISTS %s", tableName); + } + } + + /** + * INSERT OVERWRITE with {@code overwriteSchema=true} that would replace the schema of an existing + * CatalogOwned table must be blocked. + * + *

    Because {@code UCSingleCatalog} does not implement {@code StagingTableCatalog}, Spark routes + * the overwrite-with-schema-change through REPLACE TABLE AS SELECT (RTAS), which OSS Delta does + * not support. + */ + @Test + public void testInsertOverwriteWithOverwriteSchemaIsBlocked() throws Exception { + withNewTable( + "block_overwrite_schema_target", + "id INT, name STRING", + TableType.MANAGED, + targetTable -> { + sql("INSERT INTO %s VALUES (1, 'initial')", targetTable); + withNewTable( + "block_overwrite_schema_source", + "id INT, name STRING, extra STRING", + TableType.EXTERNAL, + sourceTable -> { + sql("INSERT INTO %s VALUES (2, 'new', 'extra_val')", sourceTable); + assertThrowsWithCauseContaining( + RTAS_ERROR, + () -> + spark() + .read() + .table(sourceTable) + .write() + .format("delta") + .mode("overwrite") + .option("overwriteSchema", "true") + .saveAsTable(targetTable)); + }); + }); + } + + /** + * {@code CREATE OR REPLACE TABLE} with a different schema on an existing CatalogOwned table must + * be blocked. + * + *

    Because {@code UCSingleCatalog} does not implement {@code StagingTableCatalog}, Spark routes + * {@code CREATE OR REPLACE TABLE} through REPLACE TABLE AS SELECT (RTAS), which OSS Delta does + * not support. + */ + @Test + public void testReplaceTableWithNewSchemaIsBlocked() throws Exception { + withNewTable( + "block_replace_schema_test", + "id INT, name STRING", + TableType.MANAGED, + tableName -> { + sql("INSERT INTO %s VALUES (1, 'initial')", tableName); + assertThrowsWithCauseContaining( + RTAS_ERROR, + () -> + sql( + "CREATE OR REPLACE TABLE %s (id INT, name STRING, extra STRING) " + + "USING DELTA " + + "TBLPROPERTIES ('delta.feature.catalogManaged'='supported')", + tableName)); + }); + } + + // --------------------------------------------------------------------------- + // Positive tests: operations that must still succeed + // --------------------------------------------------------------------------- + + /** Normal INSERT with no metadata change must still succeed on CatalogOwned tables. */ + @Test + public void testNormalInsertSucceedsForManagedTable() throws Exception { + withNewTable( + "normal_insert_managed_test", + "id INT, name STRING", + TableType.MANAGED, + tableName -> { + sql("INSERT INTO %s VALUES (1, 'foo'), (2, 'bar')", tableName); + check(tableName, List.of(List.of("1", "foo"), List.of("2", "bar"))); + }); + } + + /** + * INSERT with {@code autoMerge=true} but no new columns must succeed -- {@code autoMerge} only + * triggers a schema update when the incoming data actually introduces extra columns. + */ + @Test + public void testInsertWithAutoMergeAndNoSchemaChangeSucceeds() throws Exception { + withNewTable( + "auto_merge_no_change_managed_test", + "id INT, name STRING", + TableType.MANAGED, + tableName -> { + sql("INSERT INTO %s VALUES (1, 'initial')", tableName); + sql("SET spark.databricks.delta.schema.autoMerge.enabled = true"); + try { + sql("INSERT INTO %s VALUES (2, 'second')", tableName); + check(tableName, List.of(List.of("1", "initial"), List.of("2", "second"))); + } finally { + sql("SET spark.databricks.delta.schema.autoMerge.enabled = false"); + } + }); + } + + /** + * Schema evolution via INSERT with {@code autoMerge=true} must still work on EXTERNAL (non- + * CatalogOwned) tables. The kill switch must not affect tables that are not CatalogOwned. + */ + @Test + public void testInsertWithMergeSchemaStillWorksForExternalTable() throws Exception { + withNewTable( + "merge_schema_external_target", + "id INT, name STRING", + TableType.EXTERNAL, + targetTable -> { + sql("INSERT INTO %s VALUES (1, 'initial')", targetTable); + withNewTable( + "merge_schema_external_source", + "id INT, name STRING, extra STRING", + TableType.EXTERNAL, + sourceTable -> { + sql("INSERT INTO %s VALUES (2, 'new', 'extra_value')", sourceTable); + sql("SET spark.databricks.delta.schema.autoMerge.enabled = true"); + try { + // Should succeed: EXTERNAL tables are not CatalogOwned. + sql("INSERT INTO %s SELECT * FROM %s", targetTable, sourceTable); + // The target now has 3 columns; row 1 has null for 'extra'. + check( + targetTable, + List.of(List.of("1", "initial", "null"), List.of("2", "new", "extra_value"))); + } finally { + sql("SET spark.databricks.delta.schema.autoMerge.enabled = false"); + } + }); + }); + } +} diff --git a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableCreationTest.java b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableCreationTest.java index c0f66f4a163..737387d5798 100644 --- a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableCreationTest.java +++ b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableCreationTest.java @@ -27,6 +27,7 @@ import io.unitycatalog.client.model.DataSourceFormat; import io.unitycatalog.client.model.TableInfo; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -47,6 +48,7 @@ import org.apache.spark.sql.connector.catalog.TableCatalog; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DynamicTest; import org.junit.jupiter.api.Test; @@ -89,6 +91,16 @@ public class UCDeltaTableCreationTest extends UCDeltaTableIntegrationBaseTest { private static final String EXTERNAL_TBLPROPERTIES_CLAUSE = "TBLPROPERTIES ('Foo'='Bar')"; + /** + * Returns true if the Unity Catalog Spark version >0.4.0 so that it supports complex data types + * in columns and partition index. + */ + private static boolean isUcSparkNewerThan040() { + final int[] VER_0_4_0 = {0, 4, 0}; + int[] ucSparkVersion = getUnityCatalogSparkVersion(); + return Arrays.compare(ucSparkVersion, VER_0_4_0) > 0; + } + String tempDir; private Set tablesToCleanUp = new HashSet<>(); @@ -309,8 +321,11 @@ private void runTableCreationTest( if (replaceTable) { // First, create a different table to replace. sql( - "CREATE TABLE %s USING DELTA %s AS SELECT 0.1 AS col1", - fullTableName, MANAGED_TBLPROPERTIES_CLAUSE_OTHER); + "CREATE TABLE %s USING DELTA %s AS SELECT %s AS col1", + fullTableName, + MANAGED_TBLPROPERTIES_CLAUSE_OTHER, + // Older version UC Spark client can't support Decimal type + isUcSparkNewerThan040() ? "0.1" : "1"); tablesToCleanUp.add(fullTableName); } @@ -343,7 +358,8 @@ private void runTableCreationTest( comment, options.getExternalTableLocation(), withCluster, - options.getClusterColumn()); + options.getClusterColumn(), + options.getPartitionColumn()); } } @@ -437,6 +453,9 @@ public void testCreateOrReplaceTable(TableType tableType) throws Exception { @TestAllTableTypes public void testTableWithSupportedDataTypes(TableType tableType) throws Exception { + Assumptions.assumeTrue( + isUcSparkNewerThan040() || tableType != TableType.MANAGED, + "Older UC Spark package can't support uploading complex types to UC server for managed table"); String schema = // Numeric types "col_tinyint TINYINT, col_smallint SMALLINT, col_int INT, col_bigint BIGINT, " @@ -520,6 +539,9 @@ public void testTableWithSupportedDataTypes(TableType tableType) throws Exceptio @TestAllTableTypes public void testTableWithComplexTypes(TableType tableType) throws Exception { + Assumptions.assumeTrue( + isUcSparkNewerThan040() || tableType != TableType.MANAGED, + "Older UC Spark package can't support uploading complex types to UC server for managed table"); String schema = "id INT, arr ARRAY, " + "map_col MAP, " @@ -594,6 +616,7 @@ private void assertUCTableInfo( comment, externalTableLocation, false, + Optional.empty(), Optional.empty()); } @@ -605,7 +628,8 @@ private void assertUCTableInfo( String comment, String externalTableLocation, boolean withCluster, - Optional clusterColumn) + Optional clusterColumn, + Optional partitionColumn) throws ApiException { UnityCatalogInfo uc = unityCatalogInfo(); String catalogName = uc.catalogName(); @@ -628,9 +652,23 @@ private void assertUCTableInfo( // updated later and that would cause problem. List columns = tableInfo.getColumns(); assertThat(columns).isNotNull(); - assertThat(columns).isEmpty(); if (tableType == TableType.MANAGED) { + assertThat(columns).isNotEmpty(); + List columnNamesFromServer = + columns.stream().map(ColumnInfo::getName).collect(Collectors.toList()); + assertThat(columnNamesFromServer).containsExactlyInAnyOrderElementsOf(expectedColumns); + // Partition index is only set after UC-Spark 0.4.0 + if (isUcSparkNewerThan040() && partitionColumn.isPresent()) { + List matchingColumns = + columns.stream() + .filter(c -> c.getName().equals(partitionColumn.get())) + .collect(Collectors.toList()); + assertThat(matchingColumns).hasSize(1); + assertThat(matchingColumns.get(0).getPartitionIndex()).isEqualTo(0); + } else { + assertThat(columns.stream().anyMatch(c -> c.getPartitionIndex() != null)).isFalse(); + } // Delta sent properties of managed tables to server Map tablePropertiesFromServer = tableInfo.getProperties(); tablePropertiesFromServer.remove("table_type", "MANAGED"); // New property by Spark 4.1 @@ -688,6 +726,8 @@ private void assertUCTableInfo( && !expectedPropertiesWithVariableValue.contains(entry.getKey())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); assertThat(unexpectedTablePropertiesFromServer).isEmpty(); + } else { + assertThat(columns).isEmpty(); } // Also verify table using DESC EXTENDED diff --git a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameReadTest.java b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameReadTest.java new file mode 100644 index 00000000000..43a101c2ac2 --- /dev/null +++ b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameReadTest.java @@ -0,0 +1,186 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.sparkuctest; + +import static org.apache.spark.sql.functions.col; +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; +import java.util.stream.Collectors; +import org.apache.spark.sql.DataFrameReader; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * DataFrame read test suite for Delta Table operations through Unity Catalog. + * + *

    Covers spark.table(), DataFrameReader, time travel, column pruning, and filter. Most tests run + * against both EXTERNAL and MANAGED table types. + */ +public class UCDeltaTableDataFrameReadTest extends UCDeltaTableIntegrationBaseTest { + + @TestAllTableTypes + public void testReadViaSparkTable(TableType tableType) throws Exception { + withNewTable( + "df_read_spark_table", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + assertThat(ids(spark().table(tableName).orderBy("id"))).containsExactly(1, 2, 3); + }); + } + + @TestAllTableTypes + public void testReadViaDataFrameReader(TableType tableType) throws Exception { + withNewTable( + "df_read_reader", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + assertThat(ids(deltaDFReader().table(tableName).orderBy("id"))).containsExactly(1, 2, 3); + }); + } + + @TestAllTableTypes + public void testTimeTravelByVersion(TableType tableType) throws Exception { + withNewTable( + "df_time_travel_version", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + long v1 = currentVersion(tableName); + sql("INSERT INTO %s VALUES (4), (5)", tableName); + assertThat(ids(deltaDFReader().option("versionAsOf", v1).table(tableName).orderBy("id"))) + .containsExactly(1, 2, 3); + }); + } + + @TestAllTableTypes + public void testTimeTravelByTimestamp(TableType tableType) throws Exception { + withNewTable( + "df_time_travel_ts", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + String ts = currentTimestamp(tableName); + sql("INSERT INTO %s VALUES (4), (5)", tableName); + assertThat( + ids(deltaDFReader().option("timestampAsOf", ts).table(tableName).orderBy("id"))) + .containsExactly(1, 2, 3); + }); + } + + @TestAllTableTypes + public void testColumnPruning(TableType tableType) throws Exception { + withNewTable( + "df_column_pruning", + "id INT, name STRING, value INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1, 'Alice', 100), (2, 'Bob', 200)", tableName); + List rows = + spark().table(tableName).select("id", "name").orderBy("id").collectAsList(); + assertThat(rows.get(0).schema().fieldNames()).containsExactly("id", "name"); + assertThat(rows.stream().map(r -> r.getInt(0)).collect(Collectors.toList())) + .containsExactly(1, 2); + assertThat(rows.stream().map(r -> r.getString(1)).collect(Collectors.toList())) + .containsExactly("Alice", "Bob"); + }); + } + + @Test + public void testReadViaPath() throws Exception { + withNewTable( + "df_read_via_path", + "id INT", + TableType.MANAGED, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + String tablePath = + sql("DESCRIBE EXTENDED %s", tableName).stream() + .filter(row -> row.size() >= 2 && "Location".equals(row.get(0))) + .map(row -> row.get(1)) + .findFirst() + .orElseThrow(() -> new AssertionError("Could not retrieve table location")); + Assertions.assertThrows( + Exception.class, + () -> spark().read().format("delta").load(tablePath).collect(), + "Path-based access should fail for managed tables"); + }); + } + + @TestAllTableTypes + public void testChangeDataFeedViaDataFrameAPI(TableType tableType) throws Exception { + withNewTable( + "df_cdf_reader", + "id INT", + null, + tableType, + "'delta.enableChangeDataFeed'='true'", + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + sql("INSERT INTO %s VALUES (4), (5)", tableName); + long currentVersion = currentVersion(tableName); + List rows = + deltaDFReader() + .option("readChangeFeed", "true") + .option("startingVersion", currentVersion) + .table(tableName) + .orderBy("id") + .collectAsList(); + assertThat(rows.stream().map(r -> r.getInt(0)).collect(Collectors.toList())) + .containsExactly(4, 5); + }); + } + + @TestAllTableTypes + public void testEmptyTableRead(TableType tableType) throws Exception { + withNewTable( + "df_empty_read", + "id INT", + tableType, + tableName -> assertThat(spark().table(tableName).collectAsList()).isEmpty()); + } + + @TestAllTableTypes + public void testFilter(TableType tableType) throws Exception { + withNewTable( + "df_filter", + "id INT, category STRING", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1, 'A'), (2, 'B'), (3, 'A'), (4, 'B')", tableName); + assertThat( + ids(spark().table(tableName).filter(col("category").equalTo("A")).orderBy("id"))) + .containsExactly(1, 3); + }); + } + + private DataFrameReader deltaDFReader() { + return spark().read().format("delta"); + } + + private List ids(Dataset df) { + return df.collectAsList().stream().map(r -> r.getInt(0)).collect(Collectors.toList()); + } +} diff --git a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameStreamingTest.java b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameStreamingTest.java new file mode 100644 index 00000000000..c246aee7dec --- /dev/null +++ b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameStreamingTest.java @@ -0,0 +1,404 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.sparkuctest; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.spark.api.java.function.VoidFunction2; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.streaming.DataStreamReader; +import org.apache.spark.sql.streaming.StreamingQuery; +import org.apache.spark.sql.streaming.Trigger; +import org.assertj.core.api.ThrowableAssert.ThrowingCallable; +import org.junit.jupiter.api.io.TempDir; + +/** + * DataFrame streaming test suite for Delta Table operations through Unity Catalog. + * + *

    Covers streaming read and write via Structured Streaming. Uses {@link Trigger#AvailableNow()} + * for deterministic, testable streaming without manual termination. Tests run against both EXTERNAL + * and MANAGED table types. + */ +public class UCDeltaTableDataFrameStreamingTest extends UCDeltaTableIntegrationBaseTest { + + /** No-op foreachBatch sink used by negative tests that only need the stream to start. */ + private static final VoidFunction2, Long> NOOP_BATCH = (df, id) -> {}; + + @TempDir private Path tempDir; + + private int checkpointCount; + + @TestAllTableTypes + public void testStreamingReadWrite(TableType tableType) throws Exception { + withNewTable( + "streaming_rw_src", + "id INT", + tableType, + srcName -> + withNewTable( + "streaming_rw_sink", + "id INT", + tableType, + sinkName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", srcName); + String ck = checkpoint(); + + // AvailableNow: process all existing data and terminate. + spark() + .readStream() + .format("delta") + .table(srcName) + .writeStream() + .format("delta") + .outputMode("append") + .trigger(Trigger.AvailableNow()) + .option("checkpointLocation", ck) + .toTable(sinkName) + .awaitTermination(); + check(sinkName, List.of(row("1"), row("2"), row("3"))); + + // Continuous: reuse same checkpoint so the query resumes from where + // AvailableNow left off and only picks up newly inserted rows. + StreamingQuery query = + spark() + .readStream() + .format("delta") + .table(srcName) + .writeStream() + .format("delta") + .outputMode("append") + .option("checkpointLocation", ck) + .toTable(sinkName); + try { + sql("INSERT INTO %s VALUES (4), (5)", srcName); + query.processAllAvailable(); + check(sinkName, List.of(row("1"), row("2"), row("3"), row("4"), row("5"))); + } finally { + query.stop(); + } + })); + } + + @TestAllTableTypes + public void testStreamingReadFromVersion(TableType tableType) throws Exception { + withNewTable( + "streaming_version_test", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + long v1 = currentVersion(tableName); + sql("INSERT INTO %s VALUES (4), (5)", tableName); + List result = new ArrayList<>(); + spark() + .readStream() + .format("delta") + .option("startingVersion", v1 + 1) + .table(tableName) + .writeStream() + .trigger(Trigger.AvailableNow()) + .option("checkpointLocation", checkpoint()) + .foreachBatch((VoidFunction2, Long>) (df, id) -> result.addAll(ids(df))) + .start() + .awaitTermination(); + assertThat(result).containsExactlyInAnyOrder(4, 5); + }); + } + + /** + * Verifies incremental streaming via a foreachBatch sink (in-memory accumulator). Complements + * {@link #testStreamingReadWrite}, which covers the same data-arrival scenario but writes to a + * Delta table sink and validates via SQL. + */ + @TestAllTableTypes + public void testStreamingContinuous(TableType tableType) throws Exception { + withNewTable( + "streaming_continuous_test", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + List result = new ArrayList<>(); + StreamingQuery query = + spark() + .readStream() + .format("delta") + .table(tableName) + .writeStream() + .option("checkpointLocation", checkpoint()) + .foreachBatch( + (VoidFunction2, Long>) (df, id) -> result.addAll(ids(df))) + .start(); + try { + query.processAllAvailable(); + assertThat(result).containsExactlyInAnyOrder(1, 2, 3); + + sql("INSERT INTO %s VALUES (4), (5)", tableName); + query.processAllAvailable(); + assertThat(result).containsExactlyInAnyOrder(1, 2, 3, 4, 5); + } finally { + query.stop(); + } + }); + } + + /** + * Verifies that {@code maxFilesPerTrigger=1} causes the stream to process each of the 3 separate + * commits as its own micro-batch, producing exactly 3 batches total under {@link + * Trigger#AvailableNow()}. + */ + @TestAllTableTypes + public void testStreamingMaxFilesPerTrigger(TableType tableType) throws Exception { + withNewTable( + "streaming_max_files_test", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1)", tableName); + sql("INSERT INTO %s VALUES (2)", tableName); + sql("INSERT INTO %s VALUES (3)", tableName); + + List result = new ArrayList<>(); + List batchIds = new ArrayList<>(); + spark() + .readStream() + .format("delta") + .option("maxFilesPerTrigger", 1) + .table(tableName) + .writeStream() + .trigger(Trigger.AvailableNow()) + .option("checkpointLocation", checkpoint()) + .foreachBatch( + (VoidFunction2, Long>) + (df, batchId) -> { + result.addAll(ids(df)); + batchIds.add(batchId); + }) + .start() + .awaitTermination(); + assertThat(result).containsExactlyInAnyOrder(1, 2, 3); + assertThat(batchIds).hasSize(3); + }); + } + + /** + * Verifies that streaming from a table that has rows deleted (RemoveFile with dataChange=true) + * fails with a clear error directing the user to the {@code ignoreDeletes} option. + */ + @TestAllTableTypes + public void testStreamingDeleteFailsWithHelpfulError(TableType tableType) throws Exception { + withNewTable( + "streaming_delete_error_test", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + List result = new ArrayList<>(); + StreamingQuery query = + spark() + .readStream() + .format("delta") + .table(tableName) + .writeStream() + .option("checkpointLocation", checkpoint()) + .foreachBatch( + (VoidFunction2, Long>) (df, id) -> result.addAll(ids(df))) + .start(); + try { + query.processAllAvailable(); + assertThat(result).containsExactlyInAnyOrder(1, 2, 3); + + sql("DELETE FROM %s WHERE id = 1", tableName); + assertStreamingThrowsContaining(query::processAllAvailable, "ignoreDeletes"); + } finally { + query.stop(); + } + }); + } + + /** + * CDF streaming reads work for EXTERNAL tables but fail for MANAGED tables. + * + *

    For EXTERNAL: verifies that inserts and a delete produce the expected typed change events. + * For MANAGED: verifies the stream fails with an error containing "not supported" and "CDC". + */ + @TestAllTableTypes + public void testStreamingCDFRead(TableType tableType) throws Exception { + withNewTable( + "streaming_cdf_read_test", + "id INT", + null, + tableType, + "'delta.enableChangeDataFeed'='true'", + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + long insertVersion = currentVersion(tableName); + sql("DELETE FROM %s WHERE id = 1", tableName); + + if (tableType == TableType.EXTERNAL) { + List changes = new ArrayList<>(); + spark() + .readStream() + .format("delta") + .option("readChangeFeed", "true") + .option("startingVersion", insertVersion) + .table(tableName) + .writeStream() + .trigger(Trigger.AvailableNow()) + .option("checkpointLocation", checkpoint()) + .foreachBatch( + (VoidFunction2, Long>) + (df, id) -> changes.addAll(df.select("id", "_change_type").collectAsList())) + .start() + .awaitTermination(); + assertThat(changes) + .extracting(r -> r.getString(1)) + .containsExactlyInAnyOrder("insert", "insert", "insert", "delete"); + assertThat(changes) + .filteredOn(r -> "delete".equals(r.getString(1))) + .extracting(r -> r.getInt(0)) + .containsExactly(1); + } else { + assertInvalidStreamOption( + tableName, + r -> r.option("readChangeFeed", "true").option("startingVersion", insertVersion), + "not supported", + "CDC"); + } + }); + } + + /** + * Delta streaming sink does not support {@code complete} output mode. Verifies the stream fails + * with a clear error mentioning "complete". + */ + @TestAllTableTypes + public void testStreamingWriteCompleteModeNotSupported(TableType tableType) throws Exception { + withNewTable( + "streaming_complete_mode_src", + "id INT", + tableType, + srcName -> + withNewTable( + "streaming_complete_mode_sink", + "id INT", + tableType, + sinkName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", srcName); + assertStreamingThrowsContaining( + () -> + spark() + .readStream() + .format("delta") + .table(srcName) + .writeStream() + .format("delta") + .outputMode("complete") + .trigger(Trigger.AvailableNow()) + .option("checkpointLocation", checkpoint()) + .toTable(sinkName) + .awaitTermination(), + "complete"); + })); + } + + /** + * Verifies that invalid or unsupported streaming read options are rejected with clear errors. + * + *

      + *
    • Negative {@code startingVersion} is rejected (both table types). + *
    • {@code startingVersion} beyond the table history is rejected (both table types). + *
    • {@code ignoreChanges} and {@code ignoreDeletes} fail with a "not supported" error for + * MANAGED tables; EXTERNAL tables accept these options silently. + *
    + */ + @TestAllTableTypes + public void testStreamingInvalidOptions(TableType tableType) throws Exception { + withNewTable( + "streaming_invalid_options_test", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1)", tableName); + + assertInvalidStreamOption( + tableName, r -> r.option("startingVersion", -1), "startingVersion"); + assertInvalidStreamOption(tableName, r -> r.option("startingVersion", 999999), "999999"); + }); + } + + /** + * Starts a streaming read with options applied by {@code configure}, then asserts the stream + * fails with a message containing all {@code fragments} (case-insensitive). + */ + private void assertInvalidStreamOption( + String tableName, + Function configure, + String... fragments) { + assertStreamingThrowsContaining( + () -> + configure + .apply(spark().readStream().format("delta")) + .table(tableName) + .writeStream() + .trigger(Trigger.AvailableNow()) + .option("checkpointLocation", checkpoint()) + .foreachBatch(NOOP_BATCH) + .start() + .awaitTermination(), + fragments); + } + + /** + * Asserts that {@code action} throws an exception whose cause chain contains all the given {@code + * fragments} (case-insensitive). + */ + private static void assertStreamingThrowsContaining( + ThrowingCallable action, String... fragments) { + assertThatThrownBy(action) + .satisfies( + e -> { + StringBuilder full = new StringBuilder(); + for (Throwable t = e; t != null; t = t.getCause()) { + if (t.getMessage() != null) full.append(t.getMessage()).append(' '); + } + String msg = full.toString(); + for (String fragment : fragments) { + assertThat(msg).containsIgnoringCase(fragment); + } + }); + } + + private String checkpoint() throws IOException { + Path ckDir = tempDir.resolve("ck-" + checkpointCount++); + Files.createDirectory(ckDir); + return ckDir.toString(); + } + + private List ids(Dataset df) { + return df.collectAsList().stream().map(r -> r.getInt(0)).collect(Collectors.toList()); + } +} diff --git a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameWriteTest.java b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameWriteTest.java new file mode 100644 index 00000000000..70a22ce669b --- /dev/null +++ b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDataFrameWriteTest.java @@ -0,0 +1,291 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.sparkuctest; + +import static org.apache.spark.sql.functions.col; +import static org.apache.spark.sql.functions.lit; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.RowFactory; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.StructType; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.Test; + +/** + * DataFrame write test suite for Delta Table operations through Unity Catalog. + * + *

    Covers DataFrame Writer V1 (insertInto, save) and Writer V2 (writeTo) operations. Tests run + * against both EXTERNAL and MANAGED table types. + */ +public class UCDeltaTableDataFrameWriteTest extends UCDeltaTableIntegrationBaseTest { + + // Writer V1: insertInto + + @TestAllTableTypes + public void testInsertIntoAppend(TableType tableType) throws Exception { + withNewTable( + "insert_into_append_test", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + intDf(4, 5).write().mode("append").insertInto(tableName); + check(tableName, List.of(row("1"), row("2"), row("3"), row("4"), row("5"))); + }); + } + + @TestAllTableTypes + public void testInsertIntoOverwrite(TableType tableType) throws Exception { + withNewTable( + "insert_into_overwrite_test", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + intDf(9).write().mode("overwrite").insertInto(tableName); + check(tableName, List.of(row("9"))); + }); + } + + @TestAllTableTypes + public void testInsertIntoReplaceWhere(TableType tableType) throws Exception { + withNewTable( + "insert_into_replace_where_test", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + intDf(9).write().mode("overwrite").option("replaceWhere", "id > 1").insertInto(tableName); + check(tableName, List.of(row("1"), row("9"))); + }); + } + + @TestAllTableTypes + public void testSaveAsTableAppend(TableType tableType) throws Exception { + withNewTable( + "save_as_table_append_test", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + intDf(4, 5).write().format("delta").mode("append").saveAsTable(tableName); + check(tableName, List.of(row("1"), row("2"), row("3"), row("4"), row("5"))); + }); + } + + // TODO: Add saveAsTable overwrite/replaceWhere coverage once UCSingleCatalog supports REPLACE + // TABLE AS SELECT (RTAS). Currently, saveAsTable with mode("overwrite") routes through Spark's + // V2 catalog path as RTAS, which throws UnsupportedOperationException in UCSingleCatalog. + + @Test + public void testSaveByPathBlockedForManagedTable() throws Exception { + withNewTable( + "save_path_blocked_test", + "id INT", + TableType.MANAGED, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + String tablePath = + sql("DESCRIBE FORMATTED %s", tableName).stream() + .filter(r -> r.size() >= 2 && "Location".equalsIgnoreCase(r.get(0).trim())) + .map(r -> r.get(1).trim()) + .findFirst() + .orElseThrow(); + assertThatThrownBy(() -> intDf(4).write().format("delta").mode("append").save(tablePath)) + .satisfies( + e -> + assertThat(e.getMessage()) + .containsAnyOf( + "Unable to load credentials", + "DELTA_PATH_BASED_ACCESS_TO_CATALOG_MANAGED_TABLE_BLOCKED", + "Path-based access is not allowed")); + check(tableName, List.of(row("1"), row("2"), row("3"))); + }); + } + + // Writer V2: writeTo + + @TestAllTableTypes + public void testWriteToAppend(TableType tableType) throws Exception { + withNewTable( + "write_to_append_test", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + intDf(4, 5).writeTo(tableName).append(); + check(tableName, List.of(row("1"), row("2"), row("3"), row("4"), row("5"))); + }); + } + + @TestAllTableTypes + public void testWriteToOverwrite(TableType tableType) throws Exception { + withNewTable( + "write_to_overwrite_test", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + intDf(9).writeTo(tableName).overwrite(lit(true)); + check(tableName, List.of(row("9"))); + }); + } + + @TestAllTableTypes + public void testWriteToOverwriteWithCondition(TableType tableType) throws Exception { + withNewTable( + "write_to_overwrite_condition_test", + "id INT, category STRING", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1, 'A'), (2, 'B'), (3, 'A')", tableName); + spark() + .createDataFrame( + List.of(RowFactory.create(9, "A")), + new StructType() + .add("id", DataTypes.IntegerType) + .add("category", DataTypes.StringType)) + .writeTo(tableName) + .overwrite(col("category").equalTo("A")); + check(tableName, List.of(row("2", "B"), row("9", "A"))); + }); + } + + @TestAllTableTypes + public void testWriteToOverwritePartitions(TableType tableType) throws Exception { + withNewTable( + "write_to_overwrite_partitions_test", + "id INT, category STRING", + "category", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1, 'A'), (2, 'A'), (3, 'B')", tableName); + spark() + .createDataFrame( + List.of(RowFactory.create(9, "A")), + new StructType() + .add("id", DataTypes.IntegerType) + .add("category", DataTypes.StringType)) + .writeTo(tableName) + .overwritePartitions(); + // Only partition 'A' is replaced; partition 'B' remains untouched. + check(tableName, List.of(row("3", "B"), row("9", "A"))); + }); + } + + @Test + public void testWriteToCreateNewManagedTable() throws Exception { + String tableName = fullTableName("write_to_create_test"); + try { + intDf(1, 2) + .writeTo(tableName) + .using("delta") + .tableProperty("delta.feature.catalogManaged", "supported") + .create(); + check(tableName, List.of(row("1"), row("2"))); + } finally { + sql("DROP TABLE IF EXISTS %s", tableName); + } + } + + @TestAllTableTypes + public void testMergeSchema(TableType tableType) throws Exception { + Assumptions.assumeFalse( + isUCRemoteConfigured(), "mergeSchema not yet supported for UC managed tables remotely"); + if (tableType == TableType.MANAGED) { + // mergeSchema triggers updateMetadata() with a new schema, which the kill switch blocks + // on CatalogOwned tables. Assert the failure rather than skipping. + withNewTable( + "merge_schema_blocked_test", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2)", tableName); + assertThrowsWithCauseContaining( + "Metadata changes on Unity Catalog", + () -> + spark() + .createDataFrame( + List.of(RowFactory.create(3, "extra")), + new StructType() + .add("id", DataTypes.IntegerType) + .add("name", DataTypes.StringType)) + .write() + .format("delta") + .mode("append") + .option("mergeSchema", "true") + .saveAsTable(tableName)); + }); + return; + } + withNewTable( + "merge_schema_test", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2)", tableName); + spark() + .createDataFrame( + List.of(RowFactory.create(3, "extra")), + new StructType() + .add("id", DataTypes.IntegerType) + .add("name", DataTypes.StringType)) + .write() + .format("delta") + .mode("append") + .option("mergeSchema", "true") + .saveAsTable(tableName); + check(tableName, List.of(row("1", "null"), row("2", "null"), row("3", "extra"))); + }); + } + + @TestAllTableTypes + public void testWriteToPartitionedTable(TableType tableType) throws Exception { + withNewTable( + "df_partitioned_write_test", + "id INT, category STRING", + "category", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1, 'A'), (2, 'B')", tableName); + spark() + .createDataFrame( + List.of(RowFactory.create(3, "A"), RowFactory.create(4, "B")), + new StructType() + .add("id", DataTypes.IntegerType) + .add("category", DataTypes.StringType)) + .write() + .mode("append") + .insertInto(tableName); + check(tableName, List.of(row("1", "A"), row("2", "B"), row("3", "A"), row("4", "B"))); + }); + } + + private Dataset intDf(Integer... ids) { + return spark() + .createDataFrame( + Arrays.stream(ids).map(RowFactory::create).collect(Collectors.toList()), + new StructType().add("id", DataTypes.IntegerType)); + } +} diff --git a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDeltaAPITest.java b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDeltaAPITest.java new file mode 100644 index 00000000000..6904f883030 --- /dev/null +++ b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableDeltaAPITest.java @@ -0,0 +1,176 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.sparkuctest; + +import static org.apache.spark.sql.functions.col; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import io.delta.tables.DeltaTable; +import java.util.List; +import java.util.Map; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.RowFactory; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.StructType; + +/** + * Programmatic DeltaTable API test suite for Delta Table operations through Unity Catalog. + * + *

    Covers DeltaTable.forName(), update(), delete(), merge(), history(), optimize(), and restore() + * via the io.delta.tables.DeltaTable API. These go through different code paths than SQL + * equivalents tested in UCDeltaTableDMLTest and UCDeltaUtilityTest. Tests run against both EXTERNAL + * and MANAGED table types. + */ +public class UCDeltaTableDeltaAPITest extends UCDeltaTableIntegrationBaseTest { + + @TestAllTableTypes + public void testForNameAndToDF(TableType tableType) throws Exception { + withNewTable( + "dt_api_read", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + assertThat(forName(tableName).toDF().orderBy("id").collectAsList()) + .extracting(r -> r.getInt(0)) + .containsExactly(1, 2, 3); + }); + } + + @TestAllTableTypes + public void testUpdate(TableType tableType) throws Exception { + withNewTable( + "dt_api_update", + "id INT, status STRING", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1, 'pending'), (2, 'pending'), (3, 'done')", tableName); + forName(tableName).updateExpr("id = 1", Map.of("status", "'processed'")); + check(tableName, List.of(row("1", "processed"), row("2", "pending"), row("3", "done"))); + }); + } + + @TestAllTableTypes + public void testDelete(TableType tableType) throws Exception { + withNewTable( + "dt_api_delete", + "id INT, active BOOLEAN", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1, true), (2, false), (3, true)", tableName); + forName(tableName).delete(col("active").equalTo(false)); + check(tableName, List.of(row("1", "true"), row("3", "true"))); + }); + } + + @TestAllTableTypes + public void testMerge(TableType tableType) throws Exception { + withNewTable( + "dt_api_merge", + "id INT, value STRING", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1, 'old1'), (2, 'old2')", tableName); + Dataset source = + spark() + .createDataFrame( + List.of(RowFactory.create(2, "updated2"), RowFactory.create(3, "new3")), + new StructType() + .add("id", DataTypes.IntegerType) + .add("value", DataTypes.StringType)); + forName(tableName) + .as("target") + .merge(source.as("source"), "target.id = source.id") + .whenMatched() + .updateExpr(Map.of("value", "source.value")) + .whenNotMatched() + .insertExpr(Map.of("id", "source.id", "value", "source.value")) + .execute(); + check(tableName, List.of(row("1", "old1"), row("2", "updated2"), row("3", "new3"))); + }); + } + + @TestAllTableTypes + public void testHistory(TableType tableType) throws Exception { + withNewTable( + "dt_api_history", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1)", tableName); + sql("INSERT INTO %s VALUES (2)", tableName); + // CREATE TABLE (v0) + 2 INSERTs (v1, v2) + assertThat(forName(tableName).history().collectAsList()).hasSize(3); + }); + } + + @TestAllTableTypes + public void testOptimize(TableType tableType) throws Exception { + withNewTable( + "dt_api_optimize", + "id INT, category STRING", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1, 'A'), (2, 'B'), (3, 'A')", tableName); + if (tableType == TableType.MANAGED) { + assertThatThrownBy(() -> forName(tableName).optimize().executeCompaction()) + .hasMessageContaining("DELTA_UNSUPPORTED_CATALOG_MANAGED_TABLE_OPERATION"); + } else { + assertThat(forName(tableName).optimize().executeCompaction().collectAsList()) + .isNotEmpty(); + assertThat(forName(tableName).optimize().executeZOrderBy("category").collectAsList()) + .isNotEmpty(); + } + }); + } + + @TestAllTableTypes + public void testRestoreToVersion(TableType tableType) throws Exception { + withNewTable( + "dt_api_restore_version", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + long v1 = currentVersion(tableName); + sql("INSERT INTO %s VALUES (4), (5)", tableName); + forName(tableName).restoreToVersion(v1); + check(tableName, List.of(row("1"), row("2"), row("3"))); + }); + } + + @TestAllTableTypes + public void testRestoreToTimestamp(TableType tableType) throws Exception { + withNewTable( + "dt_api_restore_ts", + "id INT", + tableType, + tableName -> { + sql("INSERT INTO %s VALUES (1), (2), (3)", tableName); + String ts = currentTimestamp(tableName); + sql("INSERT INTO %s VALUES (4), (5)", tableName); + forName(tableName).restoreToTimestamp(ts); + check(tableName, List.of(row("1"), row("2"), row("3"))); + }); + } + + private DeltaTable forName(String tableName) { + return DeltaTable.forName(spark(), tableName); + } +} diff --git a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableIntegrationBaseTest.java b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableIntegrationBaseTest.java index 05533a0a2c2..0021ccf023b 100644 --- a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableIntegrationBaseTest.java +++ b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableIntegrationBaseTest.java @@ -16,10 +16,13 @@ package io.sparkuctest; +import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; + import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; +import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Arrays; @@ -32,11 +35,13 @@ import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; +import org.assertj.core.api.ThrowableAssert.ThrowingCallable; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.DynamicContainer; import org.junit.jupiter.api.DynamicTest; import org.junit.jupiter.api.TestFactory; +import org.opentest4j.TestAbortedException; /** * Abstract base class for Unity Catalog + Delta Table integration tests. @@ -75,7 +80,25 @@ Stream allTableTypesTestsFactory() { List tests = new ArrayList<>(); for (TableType tableType : ALL_TABLE_TYPES) { String testName = String.format("%s(%s)", method.getName(), tableType); - tests.add(DynamicTest.dynamicTest(testName, () -> method.invoke(this, tableType))); + tests.add( + DynamicTest.dynamicTest( + testName, + () -> { + try { + method.invoke(this, tableType); + } catch (InvocationTargetException e) { + // Unwrap so JUnit sees the original exception type. Without this, + // TestAbortedException (thrown by Assumptions) gets wrapped and JUnit + // treats the test as failed instead of skipped. Also unwrap + // RuntimeException/Error so assertThrows() in individual tests still + // matches the expected exception class rather than InvocationTargetException. + Throwable cause = e.getCause(); + if (cause instanceof TestAbortedException) throw (TestAbortedException) cause; + if (cause instanceof RuntimeException) throw (RuntimeException) cause; + if (cause instanceof Error) throw (Error) cause; + throw e; + } + })); } containers.add(DynamicContainer.dynamicContainer(method.getName(), tests)); } @@ -209,8 +232,7 @@ protected void withNewTable( String tableProperties, TestCode testCode) throws Exception { - UnityCatalogInfo uc = unityCatalogInfo(); - String fullTableName = uc.catalogName() + "." + uc.schemaName() + "." + tableName; + String fullTableName = fullTableName(tableName); // Create th partition cause. StringBuilder partitionCause = new StringBuilder(); @@ -304,6 +326,52 @@ protected void withNewTable( withNewTable(tableName, tableSchema, null, tableType, testCode); } + /** Returns the fully qualified table name for a given simple table name. */ + protected String fullTableName(String simpleName) { + UnityCatalogInfo uc = unityCatalogInfo(); + return uc.catalogName() + "." + uc.schemaName() + "." + simpleName; + } + + /** Returns the current (latest) version of the table. */ + protected long currentVersion(String tableName) { + return Long.parseLong(sql("DESCRIBE HISTORY %s LIMIT 1", tableName).get(0).get(0)); + } + + /** Returns the timestamp of the current (latest) version. */ + protected String currentTimestamp(String tableName) { + return sql("DESCRIBE HISTORY %s LIMIT 1", tableName).get(0).get(1); + } + + /** + * Asserts that the given operation throws an exception whose cause chain contains {@code + * expectedMessage}. + */ + protected void assertThrowsWithCauseContaining( + String expectedMessage, ThrowingCallable operation) { + assertThatThrownBy(operation) + .satisfies( + e -> { + Throwable t = e; + while (t != null) { + if (t.getMessage() != null && t.getMessage().contains(expectedMessage)) { + return; + } + t = t.getCause(); + } + throw new AssertionError( + "Expected exception containing '" + + expectedMessage + + "' in cause chain, but none found. Top-level: " + + e, + e); + }); + } + + /** Helper to build an expected row as a list of string values. */ + protected static List row(String... values) { + return List.of(values); + } + /** Functional interface for test code that takes a temporary directory. */ @FunctionalInterface protected interface TempDirCode { diff --git a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableReadTest.java b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableReadTest.java index 29dad0c7fd5..66a9169ff35 100644 --- a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableReadTest.java +++ b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaTableReadTest.java @@ -141,14 +141,4 @@ private void check(List> actual, List> expected) { String.format("Query results do not match.\nExpected: %s\nActual: %s", expected, actual)); } } - - /** Returns the current (latest) version of the table. */ - private long currentVersion(String tableName) { - return Long.parseLong(sql("DESCRIBE HISTORY %s LIMIT 1", tableName).get(0).get(0)); - } - - /** Returns the timestamp of the current (latest) version. */ - private String currentTimestamp(String tableName) { - return sql("DESCRIBE HISTORY %s LIMIT 1", tableName).get(0).get(1); - } } diff --git a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaUtilityTest.java b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaUtilityTest.java index a8a11fc8e07..ac9d2b6dbe9 100644 --- a/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaUtilityTest.java +++ b/spark/unitycatalog/src/test/java/io/sparkuctest/UCDeltaUtilityTest.java @@ -73,6 +73,72 @@ private void assertDescribeHistory(String tableName, List> expected Assertions.assertThat(prunedResults).isEqualTo(expected); } + @TestAllTableTypes + public void testFsPropertiesHiddenFromTableProperties(TableType tableType) throws Exception { + withNewTable( + "fs_props_hidden", + "id INT, name STRING", + null, // no partition + tableType, + "'myCustomProp'='myCustomValue'", + tableName -> { + // SHOW TBLPROPERTIES returns one row per property (key, value). + List> propRows = sql("SHOW TBLPROPERTIES %s", tableName); + List propKeys = new ArrayList<>(); + for (List row : propRows) { + propKeys.add(row.get(0)); + } + + // Verify no key starts with option.fs. — these are internal catalog-vended + // credentials/metadata that should not be user-visible. + for (String key : propKeys) { + Assertions.assertThat(key) + .as("SHOW TBLPROPERTIES should not expose option.fs.* keys") + .doesNotStartWith("option.fs."); + } + + // Verify that non-fs storage properties and user-set table properties ARE + // still present — confirming the filter is selective, not a blanket removal. + Assertions.assertThat(propKeys) + .as("User-set table properties should still be visible") + .contains("myCustomProp"); + Assertions.assertThat(propKeys) + .as("Delta table properties should still be visible") + .contains("delta.minReaderVersion"); + + // DESCRIBE EXTENDED returns a "Table Properties" row with all properties + // in a single string like "[key1=val1,key2=val2,...]". + boolean foundTableProperties = false; + List> descRows = sql("DESCRIBE EXTENDED %s", tableName); + for (List row : descRows) { + if (row.size() >= 2 && "Table Properties".equals(row.get(0))) { + foundTableProperties = true; + Assertions.assertThat(row.get(1)) + .as("DESCRIBE EXTENDED should not expose option.fs.* storage properties") + .doesNotContain("option.fs."); + Assertions.assertThat(row.get(1)) + .as("DESCRIBE EXTENDED should not expose fs.* storage properties either") + .doesNotContain("fs."); + Assertions.assertThat(row.get(1)) + .as("DESCRIBE EXTENDED should still show user-set properties") + .contains("myCustomProp=myCustomValue"); + } + } + Assertions.assertThat(foundTableProperties) + .as("DESCRIBE EXTENDED must include a 'Table Properties' row") + .isTrue(); + + // Verify the data path still works — credentials still flow to the filesystem + // via CatalogTable.storage.properties even though they are hidden from properties(). + sql("INSERT INTO %s VALUES (1, 'hello'), (2, 'world')", tableName); + check(tableName, List.of(List.of("1", "hello"), List.of("2", "world"))); + sql("INSERT INTO %s VALUES (3, 'foo')", tableName); + check( + tableName, + List.of(List.of("1", "hello"), List.of("2", "world"), List.of("3", "foo"))); + }); + } + @Test public void testMaintenanceOpsBlockedOnManagedTable() throws Exception { withNewTable( diff --git a/spark/unitycatalog/src/test/java/io/sparkuctest/UnityCatalogSupport.java b/spark/unitycatalog/src/test/java/io/sparkuctest/UnityCatalogSupport.java index 7a67cb7c346..46501f1d8a8 100644 --- a/spark/unitycatalog/src/test/java/io/sparkuctest/UnityCatalogSupport.java +++ b/spark/unitycatalog/src/test/java/io/sparkuctest/UnityCatalogSupport.java @@ -20,6 +20,7 @@ import com.google.common.collect.ImmutableMap; import io.unitycatalog.client.ApiClient; import io.unitycatalog.client.ApiClientBuilder; +import io.unitycatalog.client.VersionUtils; import io.unitycatalog.client.api.CatalogsApi; import io.unitycatalog.client.api.SchemasApi; import io.unitycatalog.client.auth.TokenProvider; @@ -337,4 +338,14 @@ public void tearDownServer() { private void deleteRecursively(File file) { FileUtils.deleteQuietly(file); } + + /** Returns Unity Catalog Spark version, like [0, 4, 0]. */ + protected static int[] getUnityCatalogSparkVersion() { + String version = Preconditions.checkNotNull(VersionUtils.VERSION); + String[] parts = version.split("[.\\-]", 4); + int major = Integer.parseInt(parts[0]); + int minor = Integer.parseInt(parts[1]); + int patch = Integer.parseInt(parts[2]); + return new int[] {major, minor, patch}; + } } diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/catalog/SparkTable.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/catalog/SparkTable.java index 4ee7b22904b..75a1be09dd7 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/catalog/SparkTable.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/catalog/SparkTable.java @@ -15,7 +15,9 @@ */ package io.delta.spark.internal.v2.catalog; +import static io.delta.spark.internal.v2.utils.ScalaUtils.toJavaOptional; import static io.delta.spark.internal.v2.utils.ScalaUtils.toScalaMap; +import static io.delta.spark.internal.v2.utils.StatsUtils.toV2Statistics; import static java.util.Objects.requireNonNull; import io.delta.kernel.Snapshot; @@ -35,6 +37,7 @@ import org.apache.spark.sql.connector.expressions.Expressions; import org.apache.spark.sql.connector.expressions.Transform; import org.apache.spark.sql.connector.read.ScanBuilder; +import org.apache.spark.sql.connector.read.Statistics; import org.apache.spark.sql.connector.write.LogicalWriteInfo; import org.apache.spark.sql.connector.write.WriteBuilder; import org.apache.spark.sql.delta.DeltaTableUtils; @@ -236,12 +239,23 @@ public ScanBuilder newScanBuilder(CaseInsensitiveStringMap scanOptions) { Map combined = new HashMap<>(this.options); combined.putAll(scanOptions.asCaseSensitiveMap()); CaseInsensitiveStringMap merged = new CaseInsensitiveStringMap(combined); + Optional catalogStats = + catalogTable + .flatMap(ct -> toJavaOptional(ct.stats())) + .map( + stats -> + toV2Statistics( + stats, + schemaProvider.getDataSchema(), + schemaProvider.getPartitionSchema())); return new SparkScanBuilder( name(), initialSnapshot, snapshotManager, schemaProvider.getDataSchema(), schemaProvider.getPartitionSchema(), + schemaProvider.getRawSchema(), + catalogStats, merged); } @@ -396,6 +410,10 @@ StructType getPartitionSchema() { return withInit(() -> partitionSchema); } + StructType getRawSchema() { + return withInit(() -> rawSchema); + } + Column[] getColumns() { return withInit(() -> columns); } diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/ddl/DDLRequestContext.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/ddl/DDLRequestContext.java new file mode 100644 index 00000000000..6269d0b1e7f --- /dev/null +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/ddl/DDLRequestContext.java @@ -0,0 +1,132 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.spark.internal.v2.ddl; + +import static java.util.Objects.requireNonNull; + +import io.delta.kernel.engine.Engine; +import io.delta.kernel.transaction.DataLayoutSpec; +import io.delta.kernel.types.StructType; +import io.delta.spark.internal.v2.snapshot.unitycatalog.UCTableInfo; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import org.apache.spark.sql.connector.catalog.Identifier; + +/** + * Immutable POJO capturing all inputs for a DDL operation (CREATE TABLE, CTAS, RTAS). + * + *

    Spark-level types (schema, partitions) are pre-converted to Kernel types by {@link + * CreateTableBuilder#prepare} so downstream code deals only with Kernel abstractions. The Spark + * {@link Identifier} is retained for logging and catalog-level identification only — it is never + * passed to Kernel APIs. + * + *

    All collection fields are defensively copied; accessors return unmodifiable views. + */ +public final class DDLRequestContext { + + private final Identifier ident; + private final String tablePath; + private final StructType kernelSchema; + private final Map properties; + private final Optional comment; + private final DataLayoutSpec dataLayoutSpec; + private final Engine engine; + private final Optional ucTableInfo; + private final String engineInfo; + + /** + * @param ident Spark catalog identifier (namespace + name); used for logging, not Kernel APIs + * @param tablePath resolved filesystem path for the Delta table + * @param kernelSchema Kernel-typed schema converted from Spark StructType + * @param properties user and Delta table properties with DSv2-internal keys already stripped + * @param comment table description from {@code CREATE TABLE ... COMMENT 'x'}, if provided; not + * yet written to the Delta log (see {@link #comment()}) + * @param dataLayoutSpec partitioning / clustering specification + * @param engine Kernel engine instance (typically {@code DefaultEngine}) + * @param ucTableInfo Unity Catalog metadata when the table is UC-managed, empty otherwise + * @param engineInfo version string for commit provenance (e.g. "Delta-Spark-DSv2/3.4.0") + */ + DDLRequestContext( + Identifier ident, + String tablePath, + StructType kernelSchema, + Map properties, + Optional comment, + DataLayoutSpec dataLayoutSpec, + Engine engine, + Optional ucTableInfo, + String engineInfo) { + this.ident = requireNonNull(ident); + this.tablePath = requireNonNull(tablePath); + this.kernelSchema = requireNonNull(kernelSchema); + this.properties = Collections.unmodifiableMap(new HashMap<>(requireNonNull(properties))); + this.comment = requireNonNull(comment); + this.dataLayoutSpec = requireNonNull(dataLayoutSpec); + this.engine = requireNonNull(engine); + this.ucTableInfo = requireNonNull(ucTableInfo); + this.engineInfo = requireNonNull(engineInfo); + } + + public Identifier ident() { + return ident; + } + + public String tablePath() { + return tablePath; + } + + public StructType kernelSchema() { + return kernelSchema; + } + + /** Returns an unmodifiable view of the table properties. */ + public Map properties() { + return properties; + } + + /** + * Table description from {@code COMMENT 'x'}, if provided. Not yet written to the Delta log + * because Kernel's {@code CreateTableTransactionBuilder} does not yet expose a {@code + * withDescription()} method. Preserved here for when that API becomes available. + */ + public Optional comment() { + return comment; + } + + public DataLayoutSpec dataLayoutSpec() { + return dataLayoutSpec; + } + + /** + * Kernel engine instance created via {@code DefaultEngine.create(hadoopConf)}. A new engine is + * created per DDL operation. {@code DefaultEngine} does not hold closeable resources so explicit + * shutdown is not required. See #5675 + * for centralizing engine creation. + */ + public Engine engine() { + return engine; + } + + public Optional ucTableInfo() { + return ucTableInfo; + } + + public String engineInfo() { + return engineInfo; + } +} diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/DeltaInputPartition.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/DeltaInputPartition.java new file mode 100644 index 00000000000..11556b9e354 --- /dev/null +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/DeltaInputPartition.java @@ -0,0 +1,94 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.spark.internal.v2.read; + +import java.util.Objects; +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.connector.read.HasPartitionKey; +import org.apache.spark.sql.connector.read.InputPartition; +import org.apache.spark.sql.execution.datasources.FilePartition; + +/** + * A Delta-specific InputPartition that wraps a FilePartition and implements HasPartitionKey. This + * enables Spark to leverage partition information for optimizations like shuffle elimination in + * joins and aggregations when using KeyGroupedPartitioning. + * + *

    Each DeltaInputPartition represents files from a single logical partition (i.e., all files + * share the same partition values). The partition key is the InternalRow containing the partition + * column values. + */ +public class DeltaInputPartition implements InputPartition, HasPartitionKey { + private final FilePartition filePartition; + private final InternalRow partitionKey; + + /** + * Creates a new DeltaInputPartition. + * + * @param filePartition The underlying FilePartition containing the files to read. + * @param partitionKey The partition key (partition column values) for all files in this + * partition. Must not be null. + */ + public DeltaInputPartition(FilePartition filePartition, InternalRow partitionKey) { + this.filePartition = Objects.requireNonNull(filePartition, "filePartition is null"); + this.partitionKey = Objects.requireNonNull(partitionKey, "partitionKey is null"); + } + + /** + * Returns the partition key (partition column values) associated with this partition. All files + * in this partition have the same partition key. + * + * @return The partition key as an InternalRow. + */ + @Override + public InternalRow partitionKey() { + return partitionKey; + } + + /** + * Returns the underlying FilePartition. + * + * @return The FilePartition containing the files to read. + */ + public FilePartition getFilePartition() { + return filePartition; + } + + @Override + public String[] preferredLocations() { + return filePartition.preferredLocations(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DeltaInputPartition that = (DeltaInputPartition) o; + return Objects.equals(filePartition, that.filePartition) + && Objects.equals(partitionKey, that.partitionKey); + } + + @Override + public int hashCode() { + return Objects.hash(filePartition, partitionKey); + } + + @Override + public String toString() { + return String.format( + "DeltaInputPartition(partitionKey=%s, files=%d)", + partitionKey, filePartition.files().length); + } +} diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkBatch.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkBatch.java index fa13f84a800..e7b1493a250 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkBatch.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkBatch.java @@ -20,10 +20,13 @@ import io.delta.spark.internal.v2.utils.PartitionUtils; import java.util.ArrayList; import java.util.Arrays; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import org.apache.hadoop.conf.Configuration; import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.connector.read.Batch; import org.apache.spark.sql.connector.read.InputPartition; import org.apache.spark.sql.connector.read.PartitionReaderFactory; @@ -86,10 +89,57 @@ public InputPartition[] planInputPartitions() { PartitionUtils.calculateMaxSplitBytes( sparkSession, totalBytes, partitionedFiles.size(), sqlConf); - scala.collection.Seq filePartitions = - FilePartition$.MODULE$.getFilePartitions( - sparkSession, JavaConverters.asScalaBuffer(partitionedFiles).toSeq(), maxSplitBytes); - return JavaConverters.seqAsJavaList(filePartitions).toArray(new InputPartition[0]); + // For non-partitioned tables, use simple file partitioning + if (partitionSchema.fields().length == 0) { + scala.collection.Seq filePartitions = + FilePartition$.MODULE$.getFilePartitions( + sparkSession, JavaConverters.asScalaBuffer(partitionedFiles).toSeq(), maxSplitBytes); + return JavaConverters.seqAsJavaList(filePartitions).toArray(new InputPartition[0]); + } + + // For partitioned tables, group files by partition values and wrap in DeltaInputPartition + // to support HasPartitionKey for KeyGroupedPartitioning optimizations + return planPartitionedInputPartitions(sparkSession, maxSplitBytes); + } + + /** + * Plans input partitions for partitioned tables by grouping files by their partition values. Each + * resulting DeltaInputPartition implements HasPartitionKey, enabling Spark to leverage partition + * information for optimizations like shuffle elimination. + */ + private InputPartition[] planPartitionedInputPartitions( + SparkSession sparkSession, long maxSplitBytes) { + // Note: Using InternalRow as map key relies on GenericInternalRow's value-based + // equals()/hashCode(), which is what PartitionUtils.getPartitionRow() returns. + Map> filesByPartition = new LinkedHashMap<>(); + for (PartitionedFile file : partitionedFiles) { + InternalRow partitionKey = file.partitionValues(); + filesByPartition.computeIfAbsent(partitionKey, k -> new ArrayList<>()).add(file); + } + + // Create DeltaInputPartitions for each partition group + List result = new ArrayList<>(); + int partitionIndex = 0; + + for (Map.Entry> entry : filesByPartition.entrySet()) { + InternalRow partitionKey = entry.getKey(); + List filesInPartition = entry.getValue(); + + // Split files within this partition based on maxSplitBytes + scala.collection.Seq filePartitions = + FilePartition$.MODULE$.getFilePartitions( + sparkSession, JavaConverters.asScalaBuffer(filesInPartition).toSeq(), maxSplitBytes); + + // Wrap each FilePartition in a DeltaInputPartition with the partition key. + // Re-index partitions with a global counter because getFilePartitions returns 0-based + // indices within each partition group, but we need unique indices across all groups. + for (FilePartition fp : JavaConverters.seqAsJavaList(filePartitions)) { + FilePartition reindexedPartition = new FilePartition(partitionIndex++, fp.files()); + result.add(new DeltaInputPartition(reindexedPartition, partitionKey)); + } + } + + return result.toArray(new InputPartition[0]); } @Override diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkMicroBatchStream.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkMicroBatchStream.java index bec8e76f217..a22b6a7c274 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkMicroBatchStream.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkMicroBatchStream.java @@ -34,6 +34,7 @@ import io.delta.kernel.internal.actions.Metadata; import io.delta.kernel.internal.actions.RemoveFile; import io.delta.kernel.internal.util.ColumnMapping; +import io.delta.kernel.internal.util.ColumnMapping.ColumnMappingMode; import io.delta.kernel.internal.util.Preconditions; import io.delta.kernel.internal.util.Utils; import io.delta.kernel.internal.util.VectorUtils; @@ -57,12 +58,14 @@ import org.apache.spark.sql.connector.read.InputPartition; import org.apache.spark.sql.connector.read.PartitionReaderFactory; import org.apache.spark.sql.connector.read.streaming.*; +import org.apache.spark.sql.delta.DeltaColumnMapping; import org.apache.spark.sql.delta.DeltaErrors; import org.apache.spark.sql.delta.DeltaOptions; import org.apache.spark.sql.delta.DeltaStartingVersion; import org.apache.spark.sql.delta.DeltaTimeTravelSpec; import org.apache.spark.sql.delta.StartingVersion; import org.apache.spark.sql.delta.StartingVersionLatest$; +import org.apache.spark.sql.delta.TypeWidening; import org.apache.spark.sql.delta.sources.DeltaSQLConf; import org.apache.spark.sql.delta.sources.DeltaSource; import org.apache.spark.sql.delta.sources.DeltaSourceOffset; @@ -84,6 +87,7 @@ import scala.collection.immutable.Seq; import scala.collection.immutable.Seq$; import scala.jdk.javaapi.CollectionConverters; +import scala.util.matching.Regex; // TODO(#5318): Use DeltaErrors error framework for consistent error handling. public class SparkMicroBatchStream @@ -98,10 +102,12 @@ public class SparkMicroBatchStream private final Engine engine; private final DeltaSnapshotManager snapshotManager; private final DeltaOptions options; + private final boolean skipChangeCommits; private final SnapshotImpl snapshotAtSourceInit; private final String tableId; private final StructType readSchemaAtSourceInit; private final boolean shouldValidateOffsets; + private final Optional excludeRegex; private final SparkSession spark; private final String tablePath; private final StructType readDataSchema; @@ -182,6 +188,7 @@ public SparkMicroBatchStream( this.spark = Objects.requireNonNull(spark, "spark is null"); this.engine = DefaultEngine.create(hadoopConf); this.options = Objects.requireNonNull(options, "options is null"); + this.skipChangeCommits = this.options.skipChangeCommits(); // Normalize tablePath to ensure it ends with "/" for consistent path construction String normalizedTablePath = Objects.requireNonNull(tablePath, "tablePath is null"); this.tablePath = @@ -207,6 +214,7 @@ public SparkMicroBatchStream( (Boolean) spark.sessionState().conf().getConf(DeltaSQLConf.STREAMING_OFFSET_VALIDATION()), "shouldValidateOffsets is null"); + this.excludeRegex = ScalaUtils.toJavaOptional(options.excludeRegex()); this.maxInitialSnapshotFiles = (Integer) spark @@ -217,7 +225,7 @@ public SparkMicroBatchStream( boolean isStreamingFromColumnMappingTable = ColumnMapping.getColumnMappingMode( this.snapshotAtSourceInit.getMetadata().getConfiguration()) - != ColumnMapping.ColumnMappingMode.NONE; + != ColumnMappingMode.NONE; boolean isTypeWideningSupportedInProtocol = this.snapshotAtSourceInit.getProtocol().supportsFeature(TYPE_WIDENING_RW_PREVIEW_FEATURE) || this.snapshotAtSourceInit.getProtocol().supportsFeature(TYPE_WIDENING_RW_FEATURE); @@ -424,6 +432,11 @@ public InputPartition[] planInputPartitions(Offset start, Offset end) { continue; } AddFile addFile = indexedFile.getAddFile(); + // TODO(#5319): Apply excludeRegex to RemoveFile/AddCDCFile when CDC is supported + if (excludeRegex.isPresent() + && excludeRegex.get().findFirstIn(addFile.getPath()).isDefined()) { + continue; + } PartitionedFile partitionedFile = PartitionUtils.buildPartitionedFile( addFile, partitionSchema, tablePath, ZoneId.of(sqlConf.sessionLocalTimeZone())); @@ -798,7 +811,7 @@ private CloseableIterator processCommitToIndexedFiles( try { long version = commit.getVersion(); - // First pass: Validate the commit. + // First pass: Validate the commit and decide whether to skip it. // // We must validate the ENTIRE commit before emitting ANY files. This is a correctness // requirement: commits could contain both AddFiles and RemoveFiles. @@ -807,13 +820,15 @@ private CloseableIterator processCommitToIndexedFiles( // // TODO(#5318): consider caching the commit actions to avoid reading the same commit twice. // TODO(#5319): don't verify metadata action when schema tracking is enabled - validateCommit( - commit, - version, - startVersion, - snapshotAtSourceInit.getPath(), - endOffsetOpt, - /* verifyMetadataAction= */ true); + boolean shouldSkipCommit = + validateCommitAndDecideSkipping( + commit, + version, + startVersion, + snapshotAtSourceInit.getPath(), + endOffsetOpt, + /* verifyMetadataAction= */ true); + // Second pass: Build a lazy iterator of IndexedFiles. // // BEGIN (BASE_INDEX) + actual file actions + END (END_INDEX) @@ -826,12 +841,22 @@ private CloseableIterator processCommitToIndexedFiles( // buildOffsetFromIndexedFile to skip re-reading completed versions. // // See DeltaSource.addBeginAndEndIndexOffsetsForVersion for the Scala equivalent. - return Utils.singletonCloseableIterator( - new IndexedFile(version, DeltaSourceOffset.BASE_INDEX(), /* addFile= */ null)) - .combine(getFilesFromCommit(commit, version)) - .combine( - Utils.singletonCloseableIterator( - new IndexedFile(version, DeltaSourceOffset.END_INDEX(), /* addFile= */ null))); + CloseableIterator fileActions = + shouldSkipCommit + ? Utils.toCloseableIterator(Collections.emptyIterator()) + : getFilesFromCommit(commit, version); + CloseableIterator inner = + Utils.singletonCloseableIterator( + new IndexedFile(version, DeltaSourceOffset.BASE_INDEX(), /* addFile= */ null)) + .combine(fileActions) + .combine( + Utils.singletonCloseableIterator( + new IndexedFile( + version, DeltaSourceOffset.END_INDEX(), /* addFile= */ null))); + + // Wrap the iterator so that closing it also closes the CommitActions, releasing its + // internal ActionsIterator and any associated file handles / parsed data. + return wrapIteratorWithCommitClose(inner, commit); } catch (Exception e) { // commit is not a CloseableIterator, we need to close it manually. Utils.closeCloseables(commit); @@ -839,6 +864,32 @@ private CloseableIterator processCommitToIndexedFiles( } } + /** + * Wraps an iterator so that closing it also closes the given {@link CommitActions}, releasing its + * internal resources. This is necessary because {@link CommitActions} is an {@link AutoCloseable} + * but not a {@link CloseableIterator}, so closing the iterator chain alone does not close the + * {@link CommitActions} that produced it. Package-private for testing. + */ + static CloseableIterator wrapIteratorWithCommitClose( + CloseableIterator inner, CommitActions commit) { + return new CloseableIterator() { + @Override + public boolean hasNext() { + return inner.hasNext(); + } + + @Override + public IndexedFile next() { + return inner.next(); + } + + @Override + public void close() throws IOException { + Utils.closeCloseables(inner, commit); + } + }; + } + private CloseableIterator getFilesFromCommit(CommitActions commit, long version) { // Assign each IndexedFile a unique index within the commit. We use a mutable array // because variables captured by a lambda must be effectively final (never reassigned). @@ -856,7 +907,7 @@ private CloseableIterator getFilesFromCommit(CommitActions commit, } /** - * Validates a commit and fail the stream if it's invalid. Mimics + * Validates a commit, fail the stream if it's invalid and decides whether to skip it. Mimics * DeltaSource.validateCommitAndDecideSkipping in Scala. * * @param commit the CommitActions representing a single commit @@ -865,9 +916,10 @@ private CloseableIterator getFilesFromCommit(CommitActions commit, * @param tablePath the path to the Delta table * @param endOffsetOpt optional end offset for boundary checking * @param verifyMetadataAction Whether to verify metadata action compatibility + * @return true if the commit should be skipped (no AddFiles emitted), false otherwise * @throws RuntimeException if the commit is invalid. */ - private void validateCommit( + private boolean validateCommitAndDecideSkipping( CommitActions commit, long version, long batchStartVersion, @@ -879,26 +931,46 @@ private void validateCommit( DeltaSourceOffset endOffset = endOffsetOpt.get(); if (endOffset.reservoirVersion() == version && endOffset.index() == DeltaSourceOffset.BASE_INDEX()) { - return; + return false; } } - // TODO(#5319): Implement ignoreChanges & skipChangeCommits & ignoreDeletes (legacy) + + // TODO(#5319): Implement ignoreFileDeletion (deprecated) + // "Changes" are commits that contain both AddFile and RemoveFile actions (e.g. UPDATE, + // MERGE that rewrites data files). Allowing changes means these commits won't cause failures. + boolean shouldAllowChanges = options.ignoreChanges() || skipChangeCommits; + // "Deletes" are commits that contain only RemoveFile actions without any AddFile actions + // (e.g. DELETE that purely removes data). This is a subset of changes, so allowing changes + // implies allowing deletes. + boolean shouldAllowDeletes = shouldAllowChanges || options.ignoreDeletes(); + + boolean hasFileAdd = false; + boolean shouldSkipCommit = false; + Metadata metadataAction = null; + String removeFileActionPath = null; try (CloseableIterator actionsIter = commit.getActions()) { while (actionsIter.hasNext()) { ColumnarBatch batch = actionsIter.next(); int numRows = batch.getSize(); - Metadata metadataAction = null; for (int rowId = 0; rowId < numRows; rowId++) { - // RULE 1: If commit has RemoveFile(dataChange=true), fail this stream. + // Track AddFile(dataChange=true) + Optional addOpt = StreamingHelper.getAddFileWithDataChange(batch, rowId); + if (addOpt.isPresent()) { + hasFileAdd = true; + } + + // Track RemoveFile(dataChange=true) Optional removeOpt = StreamingHelper.getDataChangeRemove(batch, rowId); if (removeOpt.isPresent()) { - RemoveFile removeFile = removeOpt.get(); - throw (RuntimeException) - DeltaErrors.deltaSourceIgnoreDeleteError(version, removeFile.getPath(), tablePath); + // skip change commits include delete-only commits + shouldSkipCommit = skipChangeCommits; + if (removeFileActionPath == null) { + removeFileActionPath = removeOpt.get().getPath(); + } } - // RULE 2: If commit has Metadata, check read-incompatible schema changes. + // Track Metadata for read-incompatible schema changes. Optional metadataOpt = StreamingHelper.getMetadata(batch, rowId); if (metadataOpt.isPresent()) { Metadata metadata = metadataOpt.get(); @@ -923,6 +995,21 @@ private void validateCommit( } catch (IOException e) { throw new RuntimeException("Failed to process commit at version " + version, e); } + + if (removeFileActionPath != null) { + if (hasFileAdd && !shouldAllowChanges) { + // Commit contains data changes (adds + removes) and changes are disallowed. + // TODO(#5319): log CommitInfo action's operation instead of path + throw (RuntimeException) + DeltaErrors.deltaSourceIgnoreChangesError(version, removeFileActionPath, tablePath); + } else if (!hasFileAdd && !shouldAllowDeletes) { + // Commit contains only removes (deletes) and deletes are disallowed. + throw (RuntimeException) + DeltaErrors.deltaSourceIgnoreDeleteError(version, removeFileActionPath, tablePath); + } + } + + return shouldSkipCommit; } /** @@ -967,7 +1054,32 @@ private void checkReadIncompatibleSchemaChanges( newMetadata.getId(), oldMetadata.getId()); } - // TODO(#5319): schema tracking for non-additive schema changes + // Partition column change will be ignored if user enable the unsafe flag + Seq newPartitionColumns, oldPartitionColumns; + if (schemaReadOptions.allowUnsafeStreamingReadOnPartitionColumnChanges()) { + newPartitionColumns = (Seq) Seq$.MODULE$.empty(); + oldPartitionColumns = (Seq) Seq$.MODULE$.empty(); + } else { + newPartitionColumns = + CollectionConverters.asScala( + VectorUtils.toJavaList(newMetadata.getPartitionColumns()).stream() + .map(Object::toString) + .collect(Collectors.toList())) + .toSeq(); + oldPartitionColumns = + CollectionConverters.asScala( + VectorUtils.toJavaList(oldMetadata.getPartitionColumns()).stream() + .map(Object::toString) + .collect(Collectors.toList())) + .toSeq(); + } + + checkNonAdditiveSchemaChanges( + oldMetadata, + newMetadata, + oldPartitionColumns, + newPartitionColumns, + validatedDuringStreamStart); // Other standard read compatibility changes if (!validatedDuringStreamStart @@ -980,25 +1092,6 @@ private void checkReadIncompatibleSchemaChanges( // check whether we can use `schema` (the fixed source schema we use in the same run of the // query) to read these new files safely. boolean backfilling = version < snapshotAtSourceInit.getVersion(); - // Partition column change will be ignored if user enable the unsafe flag - Seq newPartitionColumns, oldPartitionColumns; - if (schemaReadOptions.allowUnsafeStreamingReadOnPartitionColumnChanges()) { - newPartitionColumns = (Seq) Seq$.MODULE$.empty(); - oldPartitionColumns = (Seq) Seq$.MODULE$.empty(); - } else { - newPartitionColumns = - CollectionConverters.asScala( - VectorUtils.toJavaList(newMetadata.getPartitionColumns()).stream() - .map(Object::toString) - .collect(Collectors.toList())) - .toSeq(); - oldPartitionColumns = - CollectionConverters.asScala( - VectorUtils.toJavaList(oldMetadata.getPartitionColumns()).stream() - .map(Object::toString) - .collect(Collectors.toList())) - .toSeq(); - } DeltaStreamUtils.SchemaCompatibilityResult checkResult = DeltaStreamUtils.checkSchemaChangesWhenNoSchemaTracking( @@ -1024,6 +1117,68 @@ private void checkReadIncompatibleSchemaChanges( } } + // TODO(#5319): schema tracking for non-additive schema changes + // TODO(#5319): Extract the entire non-additive schema check into a static utility and share it + // with v1 by refactoring DeltaColumnMapping.hasNoColumnMappingSchemaChanges so it can be reused + // by both v1 and v2. + // Non-additive schema changes include rename column, drop column and change column type + private void checkNonAdditiveSchemaChanges( + Metadata oldMetadata, + Metadata newMetadata, + Seq oldPartitionColumns, + Seq newPartitionColumns, + boolean validatedDuringStreamStart) { + StructType sparkNewSchema = + SchemaUtils.convertKernelSchemaToSparkSchema(newMetadata.getSchema()); + StructType sparkOldSchema = + SchemaUtils.convertKernelSchemaToSparkSchema(oldMetadata.getSchema()); + + boolean shouldTrackSchema; + if (schemaReadOptions.typeWideningEnabled() + && schemaReadOptions.enableSchemaTrackingForTypeWidening() + && TypeWidening.containsWideningTypeChanges(sparkOldSchema, sparkNewSchema)) { + // If schema tracking is enabled for type widening, we will detect widening type changes and + // block the stream until the user sets `allowSourceColumnTypeChange` - similar to handling + // DROP/RENAME for column mapping. + shouldTrackSchema = true; + } else if (schemaReadOptions.allowUnsafeStreamingReadOnColumnMappingSchemaChanges()) { + shouldTrackSchema = false; + } else { + ColumnMappingMode NONE = ColumnMappingMode.NONE; + ColumnMappingMode oldMode = + ColumnMapping.getColumnMappingMode(oldMetadata.getConfiguration()); + ColumnMappingMode newMode = + ColumnMapping.getColumnMappingMode(newMetadata.getConfiguration()); + if (oldMode != NONE && newMode != NONE) { + Preconditions.checkArgument(oldMode == newMode, "changing mode is not supported"); + shouldTrackSchema = + DeltaColumnMapping.hasColMappingOrPartitionSchemaChange( + sparkNewSchema, + sparkOldSchema, + newPartitionColumns, + oldPartitionColumns, + /* isBothColumnMappingEnabled */ true); + } else if (oldMode == NONE && newMode != NONE) { + // TODO(#5319): We should disallow user to upgrade column mapping mode for now since we + // don't support schema tracking + shouldTrackSchema = true; + } else { + // Prohibit reading across a downgrade. + shouldTrackSchema = oldMode != NONE && newMode == NONE; + } + } + + if (shouldTrackSchema) { + throw (RuntimeException) + DeltaErrors.blockStreamingReadsWithIncompatibleNonAdditiveSchemaChanges( + spark, + sparkOldSchema, + sparkNewSchema, + !validatedDuringStreamStart, + /* isV2DataSource= */ true); + } + } + /** * Check read-incompatible schema changes during stream (re)start so we could fail fast. * @@ -1128,7 +1283,6 @@ private void validateSchemaCompatibilityOnStartup( } // Compare the structural schema of the analysis-time schema and snapshot schema. - // TODO(#5319): add tests for type widening, partition column, & nullability changes. if (!DataType.equalsStructurally(querySchema, snapshotSchema, /* ignoreNullability */ false)) { throw DeltaErrors.streamingSchemaMismatchOnRestart(querySchema, snapshotSchema); } diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkReaderFactory.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkReaderFactory.java index 4a951b98ab5..81ca328af39 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkReaderFactory.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkReaderFactory.java @@ -37,7 +37,7 @@ public SparkReaderFactory( @Override public PartitionReader createColumnarReader(InputPartition partition) { - return new SparkPartitionReader(readFunc, (FilePartition) partition); + return new SparkPartitionReader(readFunc, extractFilePartition(partition)); } @Override @@ -47,6 +47,23 @@ public boolean supportColumnarReads(InputPartition partition) { @Override public PartitionReader createReader(InputPartition partition) { - return new SparkPartitionReader(readFunc, (FilePartition) partition); + return new SparkPartitionReader(readFunc, extractFilePartition(partition)); + } + + /** + * Extracts the FilePartition from the given InputPartition. Handles both DeltaInputPartition (for + * partitioned tables) and FilePartition (for non-partitioned tables). + */ + private FilePartition extractFilePartition(InputPartition partition) { + if (partition instanceof DeltaInputPartition) { + return ((DeltaInputPartition) partition).getFilePartition(); + } else if (partition instanceof FilePartition) { + return (FilePartition) partition; + } else { + throw new IllegalArgumentException( + "Unexpected partition type: " + + partition.getClass().getName() + + ". Expected DeltaInputPartition or FilePartition."); + } } } diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScan.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScan.java index 6c3fceacf62..4593cd441b5 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScan.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScan.java @@ -26,6 +26,7 @@ import io.delta.kernel.internal.actions.AddFile; import io.delta.kernel.internal.data.ScanStateRow; import io.delta.kernel.utils.CloseableIterator; +import io.delta.spark.internal.v2.read.deletionvector.DeletionVectorSchemaContext; import io.delta.spark.internal.v2.snapshot.DeltaSnapshotManager; import io.delta.spark.internal.v2.utils.PartitionUtils; import io.delta.spark.internal.v2.utils.ScalaUtils; @@ -41,17 +42,27 @@ import org.apache.spark.sql.connector.expressions.FieldReference; import org.apache.spark.sql.connector.expressions.NamedReference; import org.apache.spark.sql.connector.read.*; +import org.apache.spark.sql.connector.read.colstats.ColumnStatistics; +import org.apache.spark.sql.connector.read.partitioning.KeyGroupedPartitioning; +import org.apache.spark.sql.connector.read.partitioning.Partitioning; +import org.apache.spark.sql.connector.read.partitioning.UnknownPartitioning; import org.apache.spark.sql.connector.read.streaming.MicroBatchStream; import org.apache.spark.sql.delta.DeltaOptions; import org.apache.spark.sql.execution.datasources.*; +import org.apache.spark.sql.execution.datasources.parquet.ParquetUtils; import org.apache.spark.sql.internal.SQLConf; import org.apache.spark.sql.sources.Filter; +import org.apache.spark.sql.types.StringType; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.util.CaseInsensitiveStringMap; /** Spark DSV2 Scan implementation backed by Delta Kernel. */ -public class SparkScan implements Scan, SupportsReportStatistics, SupportsRuntimeV2Filtering { +public class SparkScan + implements Scan, + SupportsReportStatistics, + SupportsRuntimeV2Filtering, + SupportsReportPartitioning { /** Supported streaming options for the V2 connector. */ private static final List SUPPORTED_STREAMING_OPTIONS = @@ -60,22 +71,23 @@ public class SparkScan implements Scan, SupportsReportStatistics, SupportsRuntim DeltaOptions.STARTING_VERSION_OPTION(), DeltaOptions.STARTING_TIMESTAMP_OPTION(), DeltaOptions.MAX_FILES_PER_TRIGGER_OPTION(), - DeltaOptions.MAX_BYTES_PER_TRIGGER_OPTION())); + DeltaOptions.MAX_BYTES_PER_TRIGGER_OPTION(), + DeltaOptions.IGNORE_CHANGES_OPTION(), + DeltaOptions.IGNORE_DELETES_OPTION(), + DeltaOptions.SKIP_CHANGE_COMMITS_OPTION(), + DeltaOptions.EXCLUDE_REGEX_OPTION())); /** * Block list of DeltaOptions that are not supported for streaming in V2 connector. Only - * startingVersion, startingTimestamp, maxFilesPerTrigger, and maxBytesPerTrigger are supported. - * User-defined custom options (not in DeltaOptions) are allowed to pass through. + * startingVersion, startingTimestamp, maxFilesPerTrigger, maxBytesPerTrigger, ignoreChanges, + * ignoreDeletes, skipChangeCommits, and excludeRegex are supported. User-defined custom options + * (not in DeltaOptions) are allowed to pass through. */ private static final Set UNSUPPORTED_STREAMING_OPTIONS = Collections.unmodifiableSet( new HashSet<>( Arrays.asList( - DeltaOptions.EXCLUDE_REGEX_OPTION().toLowerCase(), DeltaOptions.IGNORE_FILE_DELETION_OPTION().toLowerCase(), - DeltaOptions.IGNORE_CHANGES_OPTION().toLowerCase(), - DeltaOptions.IGNORE_DELETES_OPTION().toLowerCase(), - DeltaOptions.SKIP_CHANGE_COMMITS_OPTION().toLowerCase(), DeltaOptions.FAIL_ON_DATA_LOSS_OPTION().toLowerCase(), DeltaOptions.CDC_READ_OPTION().toLowerCase(), DeltaOptions.CDC_READ_OPTION_LEGACY().toLowerCase(), @@ -96,6 +108,7 @@ public class SparkScan implements Scan, SupportsReportStatistics, SupportsRuntim private final Predicate[] pushedToKernelFilters; private final Filter[] dataFilters; private final io.delta.kernel.Scan kernelScan; + private final Optional catalogStats; private final Configuration hadoopConf; private final CaseInsensitiveStringMap options; private final scala.collection.immutable.Map scalaOptions; @@ -123,6 +136,7 @@ public SparkScan( Predicate[] pushedToKernelFilters, Filter[] dataFilters, io.delta.kernel.Scan kernelScan, + Optional catalogStats, CaseInsensitiveStringMap options) { this.snapshotManager = Objects.requireNonNull(snapshotManager, "snapshotManager is null"); @@ -134,6 +148,7 @@ public SparkScan( pushedToKernelFilters == null ? new Predicate[0] : pushedToKernelFilters.clone(); this.dataFilters = dataFilters == null ? new Filter[0] : dataFilters.clone(); this.kernelScan = Objects.requireNonNull(kernelScan, "kernelScan is null"); + this.catalogStats = Objects.requireNonNull(catalogStats, "catalogStats is null"); this.options = Objects.requireNonNull(options, "options is null"); this.scalaOptions = ScalaUtils.toScalaMap(options); this.hadoopConf = SparkSession.active().sessionState().newHadoopConfWithOptions(scalaOptions); @@ -154,6 +169,39 @@ public StructType readSchema() { return new StructType(fields.toArray(new StructField[0])); } + /** + * Override columnarSupportMode to explicitly declare whether this scan supports columnar + * (vectorized) reading. Without this override, the default {@code PARTITION_DEFINED} mode causes + * Spark to eagerly call {@code planInputPartitions()} during query planning to check + * per-partition columnar support, triggering unnecessary early file enumeration. + * + *

    Since columnar support is uniform across all partitions (determined by schema compatibility + * and table features, not by individual files), we can declare it at the scan level to avoid this + * overhead. + * + *

    This must stay consistent with the vectorized reader decision in {@link + * PartitionUtils#createDeltaParquetReaderFactory}. In particular, deletion-vector-enabled tables + * augment the read schema with internal columns (e.g., {@code __delta_internal_is_row_deleted}), + * which changes the schema passed to the vectorized reader check. We replicate that logic here to + * ensure the scan-level declaration matches the per-partition reader behavior. + */ + @Override + public Scan.ColumnarSupportMode columnarSupportMode() { + // When the table supports deletion vectors, the reader factory augments the read schema + // with internal columns via DeletionVectorSchemaContext. Reuse the same class here so the + // batch-read check stays consistent — if DeletionVectorSchemaContext adds new fields in + // the future, this code path picks them up automatically. + StructType schemaForBatchCheck = + PartitionUtils.tableSupportsDeletionVectors(initialSnapshot) + ? new DeletionVectorSchemaContext(readDataSchema, partitionSchema) + .getSchemaWithDvColumn() + : readDataSchema; + + return ParquetUtils.isBatchReadSupportedForSchema(sqlConf, schemaForBatchCheck) + ? Scan.ColumnarSupportMode.SUPPORTED + : Scan.ColumnarSupportMode.UNSUPPORTED; + } + @Override public Batch toBatch() { ensurePlanned(); @@ -207,15 +255,49 @@ public String description() { @Override public Statistics estimateStatistics() { ensurePlanned(); + final long plannedBytes = estimatedSizeInBytes; + + // When catalog stats are available and CBO is enabled, combine table-level stats + // (for numRows/columnStats) with planned file stats (for sizeInBytes). + // This mirrors V1's LogicalRelation.computeStats() which gates column stats on + // conf.cboEnabled || conf.planStatsEnabled. + boolean useCatalogStats = sqlConf.cboEnabled() || sqlConf.planStatsEnabled(); + if (useCatalogStats && catalogStats.isPresent()) { + final Statistics stats = catalogStats.get(); + return new Statistics() { + @Override + public OptionalLong sizeInBytes() { + // Planned file size is authoritative (even if 0 for an empty table) + return OptionalLong.of(plannedBytes); + } + + @Override + public OptionalLong numRows() { + // TODO: Use accurate row count from planned files (sum of AddFile.numRecords) + // instead of catalog stats, which are stale (point-in-time from ANALYZE) and + // not adjusted for partition pruning. + return stats.numRows(); + } + + @Override + public Map columnStats() { + // TODO: After partition pruning, column stats (e.g. min, max, nullCount, + // distinctCount) could be tightened based on the pruned file-level stats. + return stats.columnStats(); + } + }; + } + + // No catalog stats available or CBO disabled — return stats from planned files only return new Statistics() { @Override public OptionalLong sizeInBytes() { - return OptionalLong.of(estimatedSizeInBytes); + return OptionalLong.of(plannedBytes); } @Override public OptionalLong numRows() { - // Row count is unknown at planning time. + // Row count is unknown without catalog stats return OptionalLong.empty(); } }; @@ -237,6 +319,10 @@ public OptionalLong numRows() { * (equivalent to Project output) * * + *

    When catalog column stats are available, uses per-column {@code avgLen} instead of {@code + * defaultSize()} for more accurate size estimation, mirroring {@code + * EstimationUtils.getSizePerRow()} behavior. + * *

    This provides consistent statistics with the v1 code path (LogicalRelation + visitUnaryNode * from Spark code directory). * @@ -251,16 +337,62 @@ private long computeEstimatedSizeWithColumnProjection(long totalBytes) { // Row overhead constant, matching EstimationUtils.getSizePerRow (from Spark) final int ROW_OVERHEAD = 8; - // TODO(#5952): update below with column stats from catalog when it becomes available + // Use avgLen from catalog column stats when available for more accurate estimation + Map avgLenByColumn = getAvgLenByColumn(); + final long fullSchemaRowSize = - ROW_OVERHEAD + dataSchema.defaultSize() + partitionSchema.defaultSize(); - final long outputRowSize = ROW_OVERHEAD + readSchema().defaultSize(); + ROW_OVERHEAD + + getSchemaSize(dataSchema, avgLenByColumn) + + getSchemaSize(partitionSchema, avgLenByColumn); + final long outputRowSize = ROW_OVERHEAD + getSchemaSize(readSchema(), avgLenByColumn); long estimatedBytes = (totalBytes * outputRowSize) / fullSchemaRowSize; return Math.max(1L, estimatedBytes); } + /** + * Returns a map of column name to avgLen from catalog column stats, used to improve size + * estimation accuracy when catalog stats are available. + */ + private Map getAvgLenByColumn() { + if (!catalogStats.isPresent()) { + return Collections.emptyMap(); + } + Map colStats = catalogStats.get().columnStats(); + if (colStats.isEmpty()) { + return Collections.emptyMap(); + } + Map result = new HashMap<>(); + for (Map.Entry entry : colStats.entrySet()) { + result.put(entry.getKey().fieldNames()[0], entry.getValue().avgLen()); + } + return result; + } + + /** + * Computes the estimated in-memory size for a schema, using avgLen from catalog stats when + * available, falling back to defaultSize(). Mirrors EstimationUtils.getSizePerRow(). For + * StringType columns with avgLen, adds UTF8String overhead (base + offset + numBytes = 12 bytes). + */ + private static long getSchemaSize(StructType schema, Map avgLenByColumn) { + long size = 0; + for (StructField field : schema.fields()) { + OptionalLong avgLen = avgLenByColumn.getOrDefault(field.name(), OptionalLong.empty()); + if (avgLen.isPresent()) { + if (field.dataType() instanceof StringType) { + // UTF8String: base + offset + numBytes (matching EstimationUtils.getSizePerRow) + size += avgLen.getAsLong() + 8 + 4; + } else { + size += avgLen.getAsLong(); + } + } else { + size += field.dataType().defaultSize(); + } + } + return size; + } + /** * Get the table path from the scan state. * @@ -439,6 +571,41 @@ static void validateStreamingOptions(DeltaOptions deltaOptions) { } } + /** + * Reports partition key expressions to Spark so it can recognize partition-aligned data layout. + * Called by V2ScanPartitioningAndOrdering during logical optimization to extract partition keys. + * Together with HasPartitionKey on DeltaInputPartition, this enables Spark to eliminate shuffles + * for joins and aggregations on partition columns. + * + *

    Note: This method triggers scan file materialization via {@link #ensurePlanned()} because + * {@code numPartitions} is derived from the planned file count. Since Spark calls this during + * logical optimization (before {@link #toBatch()}), this changes when planning occurs compared to + * the non-partitioned path. This is functionally correct as {@code ensurePlanned} is idempotent. + */ + @Override + public Partitioning outputPartitioning() { + // If no partition columns, return unknown partitioning + if (partitionSchema.fields().length == 0) { + return new UnknownPartitioning(0); + } + + ensurePlanned(); + + // Create partition key expressions from partition schema + org.apache.spark.sql.connector.expressions.Expression[] keys = + Arrays.stream(partitionSchema.fields()) + .map( + field -> + (org.apache.spark.sql.connector.expressions.Expression) + FieldReference.column(field.name())) + .toArray(org.apache.spark.sql.connector.expressions.Expression[]::new); + + // numPartitions is not used by Spark's KeyGroupedPartitioning handling (Spark derives + // partition count from the actual InputPartition[] with HasPartitionKey), so we use the + // file count as a reasonable upper-bound estimate. + return new KeyGroupedPartitioning(keys, partitionedFiles.size()); + } + @Override public boolean equals(Object o) { if (this == o) { @@ -458,13 +625,15 @@ public boolean equals(Object o) { // ignoring kernelScan because it is derived from Snapshot which is created from tablePath, // with pushed down filters that are also recorded in `pushedToKernelFilters` && Objects.equals(options, that.options) - && Objects.equals(appliedRuntimePredicates, that.appliedRuntimePredicates); + && Objects.equals(appliedRuntimePredicates, that.appliedRuntimePredicates) + && Objects.equals(catalogStats, that.catalogStats); } @Override public int hashCode() { int result = Objects.hash( + catalogStats, initialSnapshot.getPath(), initialSnapshot.getVersion(), dataSchema, diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScanBuilder.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScanBuilder.java index d9a6e029047..dcafbd3fecd 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScanBuilder.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/SparkScanBuilder.java @@ -25,6 +25,7 @@ import java.util.*; import java.util.stream.Collectors; import org.apache.spark.sql.connector.read.ScanBuilder; +import org.apache.spark.sql.connector.read.Statistics; import org.apache.spark.sql.connector.read.SupportsPushDownFilters; import org.apache.spark.sql.connector.read.SupportsPushDownRequiredColumns; import org.apache.spark.sql.sources.Filter; @@ -44,6 +45,8 @@ public class SparkScanBuilder private final DeltaSnapshotManager snapshotManager; private final StructType dataSchema; private final StructType partitionSchema; + private final StructType tableSchema; + private final Optional catalogStats; private final CaseInsensitiveStringMap options; private final Set partitionColumnSet; private StructType requiredDataSchema; @@ -63,6 +66,8 @@ public class SparkScanBuilder * @param snapshotManager the snapshot manager for this table * @param dataSchema the data schema (non-partition columns) * @param partitionSchema the partition schema + * @param tableSchema the full table schema (all columns) for filter type alignment + * @param catalogStats optional V2 Statistics converted from catalog stats * @param options scan options */ public SparkScanBuilder( @@ -71,12 +76,16 @@ public SparkScanBuilder( DeltaSnapshotManager snapshotManager, StructType dataSchema, StructType partitionSchema, + StructType tableSchema, + Optional catalogStats, CaseInsensitiveStringMap options) { this.initialSnapshot = requireNonNull(initialSnapshot, "initialSnapshot is null"); this.kernelScanBuilder = initialSnapshot.getScanBuilder(); this.snapshotManager = requireNonNull(snapshotManager, "snapshotManager is null"); this.dataSchema = requireNonNull(dataSchema, "dataSchema is null"); this.partitionSchema = requireNonNull(partitionSchema, "partitionSchema is null"); + this.tableSchema = requireNonNull(tableSchema, "tableSchema is null"); + this.catalogStats = requireNonNull(catalogStats, "catalogStats is null"); this.options = requireNonNull(options, "options is null"); this.requiredDataSchema = this.dataSchema; this.partitionColumnSet = @@ -106,7 +115,7 @@ public Filter[] pushFilters(Filter[] filters) { for (Filter filter : filters) { ExpressionUtils.FilterClassificationResult classification = - ExpressionUtils.classifyFilter(filter, partitionColumnSet); + ExpressionUtils.classifyFilter(filter, partitionColumnSet, tableSchema); // Collect kernel predicates if supported if (classification.isKernelSupported) { convertedKernelPredicates.add(classification.kernelPredicate.get()); @@ -167,6 +176,7 @@ public org.apache.spark.sql.connector.read.Scan build() { pushedKernelPredicates, dataFilters, kernelScanBuilder.build(), + catalogStats, options); } diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/deletionvector/ColumnVectorWithFilter.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/deletionvector/ColumnVectorWithFilter.java new file mode 100644 index 00000000000..cc8c64a77ea --- /dev/null +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/deletionvector/ColumnVectorWithFilter.java @@ -0,0 +1,152 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.spark.internal.v2.read.deletionvector; + +import org.apache.spark.sql.types.Decimal; +import org.apache.spark.sql.types.StructType; +import org.apache.spark.sql.vectorized.ColumnVector; +import org.apache.spark.sql.vectorized.ColumnarArray; +import org.apache.spark.sql.vectorized.ColumnarMap; +import org.apache.spark.unsafe.types.UTF8String; + +/** + * A column vector that applies row-level filtering using a row ID mapping. + * + *

    Wraps an existing column vector and remaps row indices during data access, effectively + * filtering the original data to only expose the live subset of rows without copying data. + * + *

    Follows Apache Iceberg's ColumnVectorWithFilter pattern. + */ +public class ColumnVectorWithFilter extends ColumnVector { + private final ColumnVector delegate; + private final int[] rowIdMapping; + private volatile ColumnVectorWithFilter[] children = null; + + public ColumnVectorWithFilter(ColumnVector delegate, int[] rowIdMapping) { + super(delegate.dataType()); + this.delegate = delegate; + this.rowIdMapping = rowIdMapping; + } + + @Override + public void close() { + delegate.close(); + } + + @Override + public boolean hasNull() { + return delegate.hasNull(); + } + + @Override + public int numNulls() { + // Computing the actual number of nulls with rowIdMapping is expensive. + // It is OK to overestimate and return the number of nulls in the original vector. + return delegate.numNulls(); + } + + @Override + public boolean isNullAt(int rowId) { + return delegate.isNullAt(rowIdMapping[rowId]); + } + + @Override + public boolean getBoolean(int rowId) { + return delegate.getBoolean(rowIdMapping[rowId]); + } + + @Override + public byte getByte(int rowId) { + return delegate.getByte(rowIdMapping[rowId]); + } + + @Override + public short getShort(int rowId) { + return delegate.getShort(rowIdMapping[rowId]); + } + + @Override + public int getInt(int rowId) { + return delegate.getInt(rowIdMapping[rowId]); + } + + @Override + public long getLong(int rowId) { + return delegate.getLong(rowIdMapping[rowId]); + } + + @Override + public float getFloat(int rowId) { + return delegate.getFloat(rowIdMapping[rowId]); + } + + @Override + public double getDouble(int rowId) { + return delegate.getDouble(rowIdMapping[rowId]); + } + + @Override + public ColumnarArray getArray(int rowId) { + return delegate.getArray(rowIdMapping[rowId]); + } + + @Override + public ColumnarMap getMap(int rowId) { + return delegate.getMap(rowIdMapping[rowId]); + } + + @Override + public Decimal getDecimal(int rowId, int precision, int scale) { + return delegate.getDecimal(rowIdMapping[rowId], precision, scale); + } + + @Override + public UTF8String getUTF8String(int rowId) { + return delegate.getUTF8String(rowIdMapping[rowId]); + } + + @Override + public byte[] getBinary(int rowId) { + return delegate.getBinary(rowIdMapping[rowId]); + } + + /** + * Returns the child vector at the given ordinal, wrapped with the same row ID mapping. + * + *

    Uses double-checked locking for thread-safe lazy initialization: the volatile {@code + * children} field allows lock-free reads after initialization, while the synchronized block + * ensures at-most-once creation. All children are created eagerly within the lock to avoid + * per-element races. + */ + @Override + public ColumnVector getChild(int ordinal) { + if (children == null) { + synchronized (this) { + if (children == null) { + // Eagerly create all children to avoid race condition on children[ordinal] access + StructType structType = (StructType) dataType(); + ColumnVectorWithFilter[] newChildren = + new ColumnVectorWithFilter[structType.fields().length]; + for (int i = 0; i < newChildren.length; i++) { + newChildren[i] = new ColumnVectorWithFilter(delegate.getChild(i), rowIdMapping); + } + children = newChildren; + } + } + } + return children[ordinal]; + } +} diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/deletionvector/DeletionVectorReadFunction.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/deletionvector/DeletionVectorReadFunction.java index 3a0f3718400..77e93e80dc9 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/read/deletionvector/DeletionVectorReadFunction.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/read/deletionvector/DeletionVectorReadFunction.java @@ -17,9 +17,12 @@ import io.delta.spark.internal.v2.utils.CloseableIterator; import java.io.Serializable; +import java.util.Arrays; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.catalyst.ProjectingInternalRow; import org.apache.spark.sql.execution.datasources.PartitionedFile; +import org.apache.spark.sql.vectorized.ColumnVector; +import org.apache.spark.sql.vectorized.ColumnarBatch; import scala.Function1; import scala.collection.Iterator; import scala.runtime.AbstractFunction1; @@ -37,6 +40,9 @@ * *

    The returned iterator implements {@link java.io.Closeable} to ensure proper resource cleanup * of the underlying Parquet reader, even when the iterator is not fully consumed. + * + *

    The reader mode (row-based vs vectorized) is determined once at scan planning time and does + * not change mid-stream. See {@link #wrap}. */ public class DeletionVectorReadFunction extends AbstractFunction1> implements Serializable { @@ -48,27 +54,34 @@ public class DeletionVectorReadFunction private final Function1> baseReadFunc; private final DeletionVectorSchemaContext dvSchemaContext; + private final boolean isVectorizedReader; private DeletionVectorReadFunction( Function1> baseReadFunc, - DeletionVectorSchemaContext dvSchemaContext) { + DeletionVectorSchemaContext dvSchemaContext, + boolean isVectorizedReader) { this.baseReadFunc = baseReadFunc; this.dvSchemaContext = dvSchemaContext; + this.isVectorizedReader = isVectorizedReader; } @Override public Iterator apply(PartitionedFile file) { + if (isVectorizedReader) { + return applyBatch(file); + } else { + return applyRow(file); + } + } + + /** Row-based: filter deleted rows and project out the DV column. */ + private Iterator applyRow(PartitionedFile file) { int dvColumnIndex = dvSchemaContext.getDvColumnIndex(); - // Use pre-computed ordinals from DeletionVectorSchemaContext. ProjectingInternalRow projection = ProjectingInternalRow.apply( dvSchemaContext.getOutputSchema(), dvSchemaContext.getOutputColumnOrdinals()); - // Wrap the base iterator as CloseableIterator to preserve close() through filter/map. - // This ensures proper resource cleanup even when the iterator is not fully consumed. - Iterator baseIterator = baseReadFunc.apply(file); - - return CloseableIterator.wrap(baseIterator) + return CloseableIterator.wrap(baseReadFunc.apply(file)) .filterCloseable(row -> row.getByte(dvColumnIndex) == ROW_NOT_DELETED) .mapCloseable( row -> { @@ -77,10 +90,70 @@ public Iterator apply(PartitionedFile file) { }); } + /** Vectorized: filter active rows and project out the DV column from each batch. */ + @SuppressWarnings("unchecked") + private Iterator applyBatch(PartitionedFile file) { + int dvColumnIndex = dvSchemaContext.getDvColumnIndex(); + + // In vectorized mode Spark passes ColumnarBatch via type erasure as InternalRow. + Iterator baseIterator = (Iterator) (Iterator) baseReadFunc.apply(file); + return (Iterator) + (Iterator) + CloseableIterator.wrap(baseIterator) + .mapCloseable( + item -> { + if (item instanceof ColumnarBatch) { + return filterAndProjectBatch((ColumnarBatch) item, dvColumnIndex); + } + throw new IllegalStateException( + "Expected ColumnarBatch when vectorized reader is enabled, but got: " + + item.getClass()); + }); + } + + /** Filter active rows and project out the DV column from a ColumnarBatch. */ + private static ColumnarBatch filterAndProjectBatch(ColumnarBatch batch, int dvColumnIndex) { + int[] activeRows = findActiveRows(batch, dvColumnIndex); + ColumnVector[] filteredColumns = buildFilteredColumns(batch, dvColumnIndex, activeRows); + return new ColumnarBatch(filteredColumns, activeRows.length); + } + + /** + * Build projected output columns by dropping the DV column and applying active row mapping. The + * excluded DV column's lifecycle is managed by the base Spark vectorized reader, not by us. + */ + private static ColumnVector[] buildFilteredColumns( + ColumnarBatch batch, int dvColumnIndex, int[] activeRows) { + ColumnVector[] filteredColumns = new ColumnVector[batch.numCols() - 1]; + int outputIndex = 0; + for (int inputIndex = 0; inputIndex < batch.numCols(); inputIndex++) { + if (inputIndex == dvColumnIndex) { + continue; + } + filteredColumns[outputIndex++] = + new ColumnVectorWithFilter(batch.column(inputIndex), activeRows); + } + return filteredColumns; + } + + /** Find indices of rows where DV column is 0 (not deleted). */ + private static int[] findActiveRows(ColumnarBatch batch, int dvColumnIndex) { + ColumnVector dvColumn = batch.column(dvColumnIndex); + int[] temp = new int[batch.numRows()]; + int count = 0; + for (int i = 0; i < batch.numRows(); i++) { + if (dvColumn.getByte(i) == ROW_NOT_DELETED) { + temp[count++] = i; + } + } + return Arrays.copyOf(temp, count); + } + /** Factory method to wrap a reader function with DV filtering. */ public static DeletionVectorReadFunction wrap( Function1> baseReadFunc, - DeletionVectorSchemaContext dvSchemaContext) { - return new DeletionVectorReadFunction(baseReadFunc, dvSchemaContext); + DeletionVectorSchemaContext dvSchemaContext, + boolean isVectorizedReader) { + return new DeletionVectorReadFunction(baseReadFunc, dvSchemaContext, isVectorizedReader); } } diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/ExpressionUtils.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/ExpressionUtils.java index 7343834ca27..38674f38e1b 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/ExpressionUtils.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/ExpressionUtils.java @@ -18,12 +18,16 @@ import static org.apache.spark.sql.connector.catalog.CatalogV2Implicits.parseColumnPath; import com.google.common.annotations.VisibleForTesting; +import io.delta.kernel.expressions.AlwaysFalse; +import io.delta.kernel.expressions.AlwaysTrue; import io.delta.kernel.expressions.And; import io.delta.kernel.expressions.Column; +import io.delta.kernel.expressions.In; import io.delta.kernel.expressions.Literal; import io.delta.kernel.expressions.Or; import io.delta.kernel.expressions.Predicate; import io.delta.kernel.internal.util.InternalUtils; +import java.math.BigDecimal; import java.sql.Date; import java.sql.Timestamp; import java.util.*; @@ -32,6 +36,8 @@ import org.apache.spark.sql.connector.expressions.LiteralValue; import org.apache.spark.sql.connector.expressions.NamedReference; import org.apache.spark.sql.sources.*; +import org.apache.spark.sql.types.DataType; +import org.apache.spark.sql.types.DecimalType; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; import org.apache.spark.unsafe.types.UTF8String; @@ -56,6 +62,8 @@ public final class ExpressionUtils { *
  • Comparison: EqualTo, GreaterThan, LessThan, etc. *
  • Null tests: IsNull, IsNotNull *
  • Null-safe comparison: EqualNullSafe + *
  • String prefix: StringStartsWith + *
  • Set membership: In *
  • Logical operators: And, Or, Not * * @@ -64,7 +72,25 @@ public final class ExpressionUtils { * not supported, along with a boolean indicating whether the conversion was partial */ public static ConvertedPredicate convertSparkFilterToKernelPredicate(Filter filter) { - return convertSparkFilterToKernelPredicate(filter, true /*canPartialPushDown*/); + return convertSparkFilterToKernelPredicate(filter, true /*canPartialPushDown*/, null); + } + + /** + * Converts a Spark SQL filter to a Delta Kernel predicate with table schema for type alignment. + * + *

    When a table schema is provided, decimal literal types in comparison filters are widened to + * match the column's declared decimal type. This prevents type mismatch errors in the Kernel's + * expression evaluator during data skipping, where stats column types (from the table schema) + * must match the filter literal types exactly. + * + * @param filter the Spark SQL filter to convert + * @param tableSchema the table schema for looking up column types, may be null + * @return ConvertedPredicate containing the converted Kernel predicate, or empty if conversion is + * not supported, along with a boolean indicating whether the conversion was partial + */ + public static ConvertedPredicate convertSparkFilterToKernelPredicate( + Filter filter, StructType tableSchema) { + return convertSparkFilterToKernelPredicate(filter, true /*canPartialPushDown*/, tableSchema); } /** @@ -80,10 +106,23 @@ public static ConvertedPredicate convertSparkFilterToKernelPredicate(Filter filt @VisibleForTesting static ConvertedPredicate convertSparkFilterToKernelPredicate( Filter filter, boolean canPartialPushDown) { + return convertSparkFilterToKernelPredicate(filter, canPartialPushDown, null); + } + + /** + * Core implementation of filter-to-predicate conversion with partial pushdown control and + * optional table schema for decimal type alignment. + * + * @param filter the Spark SQL filter to convert + * @param canPartialPushDown whether partial pushdown is allowed for AND filters + * @param tableSchema the table schema for looking up column types, may be null + */ + private static ConvertedPredicate convertSparkFilterToKernelPredicate( + Filter filter, boolean canPartialPushDown, StructType tableSchema) { if (filter instanceof EqualTo) { EqualTo f = (EqualTo) filter; return new ConvertedPredicate( - convertValueToKernelLiteral(f.value()) + convertComparisonLiteral(f.value(), f.attribute(), tableSchema) .map(l -> new Predicate("=", kernelColumn(f.attribute()), l))); } if (filter instanceof EqualNullSafe) { @@ -93,31 +132,31 @@ static ConvertedPredicate convertSparkFilterToKernelPredicate( return new ConvertedPredicate( f.value() == null ? Optional.of(new Predicate("IS_NULL", kernelColumn(f.attribute()))) - : convertValueToKernelLiteral(f.value()) + : convertComparisonLiteral(f.value(), f.attribute(), tableSchema) .map(l -> new Predicate("=", kernelColumn(f.attribute()), l))); } if (filter instanceof GreaterThan) { GreaterThan f = (GreaterThan) filter; return new ConvertedPredicate( - convertValueToKernelLiteral(f.value()) + convertComparisonLiteral(f.value(), f.attribute(), tableSchema) .map(l -> new Predicate(">", kernelColumn(f.attribute()), l))); } if (filter instanceof GreaterThanOrEqual) { GreaterThanOrEqual f = (GreaterThanOrEqual) filter; return new ConvertedPredicate( - convertValueToKernelLiteral(f.value()) + convertComparisonLiteral(f.value(), f.attribute(), tableSchema) .map(l -> new Predicate(">=", kernelColumn(f.attribute()), l))); } if (filter instanceof LessThan) { LessThan f = (LessThan) filter; return new ConvertedPredicate( - convertValueToKernelLiteral(f.value()) + convertComparisonLiteral(f.value(), f.attribute(), tableSchema) .map(l -> new Predicate("<", kernelColumn(f.attribute()), l))); } if (filter instanceof LessThanOrEqual) { LessThanOrEqual f = (LessThanOrEqual) filter; return new ConvertedPredicate( - convertValueToKernelLiteral(f.value()) + convertComparisonLiteral(f.value(), f.attribute(), tableSchema) .map(l -> new Predicate("<=", kernelColumn(f.attribute()), l))); } if (filter instanceof IsNull) { @@ -130,10 +169,37 @@ static ConvertedPredicate convertSparkFilterToKernelPredicate( return new ConvertedPredicate( Optional.of(new Predicate("IS_NOT_NULL", kernelColumn(f.attribute())))); } + if (filter instanceof StringStartsWith) { + StringStartsWith f = (StringStartsWith) filter; + return new ConvertedPredicate( + convertValueToKernelLiteral(f.value()) + .map(l -> new Predicate("STARTS_WITH", kernelColumn(f.attribute()), l))); + } + if (filter instanceof org.apache.spark.sql.sources.In) { + org.apache.spark.sql.sources.In f = (org.apache.spark.sql.sources.In) filter; + // An empty IN list can never match any row. Push ALWAYS_FALSE so the kernel skips + // all files entirely, rather than scanning every file only to discard every row. + if (f.values().length == 0) { + return new ConvertedPredicate(Optional.of(AlwaysFalse.ALWAYS_FALSE)); + } + List literals = new ArrayList<>(); + for (Object value : f.values()) { + Optional lit = convertValueToKernelLiteral(value); + if (!lit.isPresent()) { + // A value that can't be converted (e.g. null, unsupported type) makes the whole + // IN expression unsafe to push down; return empty to keep it for post-scan evaluation. + return new ConvertedPredicate(Optional.empty()); + } + literals.add(lit.get()); + } + return new ConvertedPredicate(Optional.of(new In(kernelColumn(f.attribute()), literals))); + } if (filter instanceof org.apache.spark.sql.sources.And) { org.apache.spark.sql.sources.And f = (org.apache.spark.sql.sources.And) filter; - ConvertedPredicate left = convertSparkFilterToKernelPredicate(f.left(), canPartialPushDown); - ConvertedPredicate right = convertSparkFilterToKernelPredicate(f.right(), canPartialPushDown); + ConvertedPredicate left = + convertSparkFilterToKernelPredicate(f.left(), canPartialPushDown, tableSchema); + ConvertedPredicate right = + convertSparkFilterToKernelPredicate(f.right(), canPartialPushDown, tableSchema); boolean isPartial = left.isPartial() || right.isPartial(); if (left.isPresent() && right.isPresent()) { return new ConvertedPredicate(Optional.of(new And(left.get(), right.get())), isPartial); @@ -148,8 +214,10 @@ static ConvertedPredicate convertSparkFilterToKernelPredicate( } if (filter instanceof org.apache.spark.sql.sources.Or) { org.apache.spark.sql.sources.Or f = (org.apache.spark.sql.sources.Or) filter; - ConvertedPredicate left = convertSparkFilterToKernelPredicate(f.left(), canPartialPushDown); - ConvertedPredicate right = convertSparkFilterToKernelPredicate(f.right(), canPartialPushDown); + ConvertedPredicate left = + convertSparkFilterToKernelPredicate(f.left(), canPartialPushDown, tableSchema); + ConvertedPredicate right = + convertSparkFilterToKernelPredicate(f.right(), canPartialPushDown, tableSchema); // OR requires both operands to be convertible for correctness boolean isPartial = left.isPartial() || right.isPartial(); if (!left.isPresent() || !right.isPresent()) { @@ -178,14 +246,81 @@ static ConvertedPredicate convertSparkFilterToKernelPredicate( // NOT(age < 30) = NOT(true) = false → system excludes both row // We will return incorrect result, then. ConvertedPredicate child = - convertSparkFilterToKernelPredicate(f.child(), false /*canPartialPushDown*/); + convertSparkFilterToKernelPredicate(f.child(), false /*canPartialPushDown*/, tableSchema); return new ConvertedPredicate( child.getConvertedPredicate().map(c -> new Predicate("NOT", c)), child.isPartial()); } + if (filter instanceof org.apache.spark.sql.sources.AlwaysTrue) { + return new ConvertedPredicate(Optional.of(AlwaysTrue.ALWAYS_TRUE)); + } + if (filter instanceof org.apache.spark.sql.sources.AlwaysFalse) { + return new ConvertedPredicate(Optional.of(AlwaysFalse.ALWAYS_FALSE)); + } + return new ConvertedPredicate(Optional.empty()); } + /** + * Converts a filter literal value to a Kernel Literal, aligning decimal types with the column's + * declared type from the table schema when available. + * + *

    When the value is a {@link java.math.BigDecimal} and the table schema is provided, this + * method looks up the column's declared {@link org.apache.spark.sql.types.DecimalType} and widens + * the literal to match. This prevents type mismatch errors in the Kernel's expression evaluator + * during data skipping, where stats column types must match the filter literal types exactly. + * + *

    If the literal's scale exceeds the column's scale, or the widened value exceeds the column's + * precision, the filter cannot be safely pushed down and an empty Optional is returned. + * + * @param value the filter literal value + * @param attribute the column name referenced by the filter + * @param tableSchema the table schema for type lookup, may be null + * @return Optional containing the Kernel Literal with aligned type, or empty if conversion fails + */ + private static Optional convertComparisonLiteral( + Object value, String attribute, StructType tableSchema) { + if (value instanceof BigDecimal && tableSchema != null) { + BigDecimal bd = (BigDecimal) value; + Optional columnType = lookupColumnType(attribute, tableSchema); + if (columnType.isPresent() && columnType.get() instanceof DecimalType) { + DecimalType colDecimalType = (DecimalType) columnType.get(); + return widenDecimalLiteral(bd, colDecimalType.precision(), colDecimalType.scale()); + } + } + return convertValueToKernelLiteral(value); + } + + /** + * Widens a BigDecimal literal to match the target decimal precision and scale. Returns empty if + * the literal cannot be safely represented in the target type (e.g., the literal has more decimal + * digits than the target scale, or the widened value exceeds the target precision). + */ + private static Optional widenDecimalLiteral( + BigDecimal bd, int targetPrecision, int targetScale) { + if (bd.scale() <= targetScale) { + BigDecimal widened = bd.setScale(targetScale); + if (widened.precision() <= targetPrecision) { + return Optional.of(Literal.ofDecimal(widened, targetPrecision, targetScale)); + } + } + // Literal doesn't fit in column type or has higher scale - skip pushdown + return Optional.empty(); + } + + /** + * Looks up the data type of a top-level column in the table schema using case-insensitive name + * matching. Returns empty for nested columns or if the column is not found. + */ + private static Optional lookupColumnType(String attribute, StructType tableSchema) { + for (StructField field : tableSchema.fields()) { + if (field.name().equalsIgnoreCase(attribute)) { + return Optional.of(field.dataType()); + } + } + return Optional.empty(); + } + /** * Creates a Delta Kernel Column from a Spark SQL column attribute name. * @@ -267,10 +402,12 @@ static Optional convertValueToKernelLiteral(Object value) { Double d = (Double) value; return Optional.of(Literal.ofDouble(d)); } - if (value instanceof java.math.BigDecimal) { - // Preserve precision and scale from the original BigDecimal - java.math.BigDecimal bd = (java.math.BigDecimal) value; - return Optional.of(Literal.ofDecimal(bd, bd.precision(), bd.scale())); + if (value instanceof BigDecimal) { + BigDecimal bd = (BigDecimal) value; + // BigDecimal precision can be less than scale (e.g. BigDecimal("0.00") has + // precision=1, scale=2). Kernel requires precision >= scale, so normalize. + int precision = Math.max(bd.precision(), bd.scale() + 1); + return Optional.of(Literal.ofDecimal(bd, precision, bd.scale())); } if (value instanceof UTF8String) { UTF8String s = (UTF8String) value; @@ -373,9 +510,23 @@ public FilterClassificationResult( */ public static FilterClassificationResult classifyFilter( Filter filter, Set partitionColumnSet) { + return classifyFilter(filter, partitionColumnSet, null); + } + + /** + * Classifies a Spark Filter with table schema for decimal type alignment. + * + * @param filter the Spark Filter to classify + * @param partitionColumnSet a set of partition column names (in lower case) for identifying data + * filters + * @param tableSchema the table schema for aligning decimal literal types, may be null + * @return FilterClassificationResult containing classification details + */ + public static FilterClassificationResult classifyFilter( + Filter filter, Set partitionColumnSet, StructType tableSchema) { // try to convert Spark filter to Kernel Predicate ConvertedPredicate convertedPredicate = - ExpressionUtils.convertSparkFilterToKernelPredicate(filter); + ExpressionUtils.convertSparkFilterToKernelPredicate(filter, tableSchema); boolean isKernelSupported = convertedPredicate.isPresent(); boolean isPartialConversion = convertedPredicate.isPartial(); diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/PartitionUtils.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/PartitionUtils.java index 6dc0b7b3193..6c6e067b340 100644 --- a/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/PartitionUtils.java +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/PartitionUtils.java @@ -60,6 +60,19 @@ public class PartitionUtils { private PartitionUtils() {} + /** + * Returns whether the given snapshot's table supports deletion vectors. A table supports DVs when + * its protocol includes the {@link TableFeatures#DELETION_VECTORS_RW_FEATURE} and the table + * format is Parquet. + */ + public static boolean tableSupportsDeletionVectors(Snapshot snapshot) { + SnapshotImpl snapshotImpl = (SnapshotImpl) snapshot; + Protocol protocol = snapshotImpl.getProtocol(); + Metadata metadata = snapshotImpl.getMetadata(); + return protocol.supportsFeature(TableFeatures.DELETION_VECTORS_RW_FEATURE) + && "parquet".equalsIgnoreCase(metadata.getFormat().getProvider()); + } + /** * Calculate the maximum split bytes for file partitioning, considering total bytes and file * count. This is used for optimal file splitting in both batch and streaming read. @@ -93,6 +106,12 @@ public static long calculateMaxSplitBytes( *

    Note: Partition values in AddFile use physical column names as keys when column mapping is * enabled. This method uses DeltaColumnMapping.getPhysicalName to map from logical schema fields * to physical partition value keys. + * + * @implNote The returned {@link InternalRow} is a {@code GenericInternalRow} (via {@code + * InternalRow.fromSeq}), which has value-based {@code equals()}/{@code hashCode()}. Callers + * such as {@code SparkBatch.planPartitionedInputPartitions} rely on this for grouping files + * by partition key. Changing the return type to a different InternalRow subtype (e.g. {@code + * UnsafeRow}) may break that contract. */ public static InternalRow getPartitionRow( MapValue partitionValues, StructType partitionSchema, ZoneId zoneId) { @@ -199,11 +218,8 @@ public static PartitionReaderFactory createDeltaParquetReaderFactory( String tablePath = snapshotImpl.getDataPath().toString(); // Create DV schema context if table supports deletion vectors - boolean isTableSupportDv = - protocol.supportsFeature(TableFeatures.DELETION_VECTORS_RW_FEATURE) - && "parquet".equalsIgnoreCase(metadata.getFormat().getProvider()); Optional dvSchemaContext = - isTableSupportDv + tableSupportsDeletionVectors(snapshot) ? Optional.of(new DeletionVectorSchemaContext(readDataSchema, partitionSchema)) : Optional.empty(); StructType finalReadDataSchema = @@ -211,9 +227,8 @@ public static PartitionReaderFactory createDeltaParquetReaderFactory( .map(DeletionVectorSchemaContext::getSchemaWithDvColumn) .orElse(readDataSchema); - // TODO(https://github.com/delta-io/delta/issues/5859): Enable vectorized reader for DV tables boolean enableVectorizedReader = - !isTableSupportDv && ParquetUtils.isBatchReadSupportedForSchema(sqlConf, readDataSchema); + ParquetUtils.isBatchReadSupportedForSchema(sqlConf, finalReadDataSchema); scala.collection.immutable.Map optionsWithVectorizedReading = scalaOptions.$plus( new Tuple2<>( @@ -221,11 +236,11 @@ public static PartitionReaderFactory createDeltaParquetReaderFactory( String.valueOf(enableVectorizedReader))); // TODO(https://github.com/delta-io/delta/issues/5859): Enable file splitting for DV tables - boolean optimizationsEnabled = !isTableSupportDv; + boolean optimizationsEnabled = !dvSchemaContext.isPresent(); // TODO(https://github.com/delta-io/delta/issues/5859): Support _metadata.row_index for DV Option useMetadataRowIndex = - isTableSupportDv ? Option.apply(Boolean.FALSE) : Option.empty(); + dvSchemaContext.isPresent() ? Option.apply(Boolean.FALSE) : Option.empty(); DeltaParquetFileFormatV2 deltaFormat = new DeltaParquetFileFormatV2( protocol, @@ -249,7 +264,8 @@ public static PartitionReaderFactory createDeltaParquetReaderFactory( // Wrap reader to filter deleted rows and remove internal columns if DV is enabled. if (dvSchemaContext.isPresent()) { - readFunc = DeletionVectorReadFunction.wrap(readFunc, dvSchemaContext.get()); + readFunc = + DeletionVectorReadFunction.wrap(readFunc, dvSchemaContext.get(), enableVectorizedReader); } return new SparkReaderFactory(readFunc, enableVectorizedReader); diff --git a/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/StatsUtils.java b/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/StatsUtils.java new file mode 100644 index 00000000000..cc113b79b63 --- /dev/null +++ b/spark/v2/src/main/java/io/delta/spark/internal/v2/utils/StatsUtils.java @@ -0,0 +1,170 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.spark.internal.v2.utils; + +import static io.delta.spark.internal.v2.utils.ScalaUtils.toJavaOptional; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import org.apache.spark.sql.catalyst.catalog.CatalogColumnStat; +import org.apache.spark.sql.catalyst.catalog.CatalogColumnStat$; +import org.apache.spark.sql.catalyst.catalog.CatalogStatistics; +import org.apache.spark.sql.connector.expressions.FieldReference; +import org.apache.spark.sql.connector.expressions.NamedReference; +import org.apache.spark.sql.connector.read.Statistics; +import org.apache.spark.sql.connector.read.colstats.ColumnStatistics; +import org.apache.spark.sql.types.DataType; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; + +/** Utilities for converting catalog statistics to V2 connector statistics. */ +public final class StatsUtils { + + private StatsUtils() {} + + /** + * Convert {@link CatalogStatistics} to V2 connector {@link Statistics}. + * + * @param catalogStats the catalog statistics to convert + * @param dataSchema the data schema (non-partition columns) + * @param partitionSchema the partition schema + * @return V2 Statistics representation + */ + public static Statistics toV2Statistics( + CatalogStatistics catalogStats, StructType dataSchema, StructType partitionSchema) { + // Build a map of column name -> DataType from both schemas + Map columnTypes = new HashMap<>(); + for (StructField field : dataSchema.fields()) { + columnTypes.put(field.name(), field.dataType()); + } + for (StructField field : partitionSchema.fields()) { + columnTypes.put(field.name(), field.dataType()); + } + + Map colStatsMap = buildColumnStats(catalogStats, columnTypes); + + return new Statistics() { + @Override + public OptionalLong sizeInBytes() { + return OptionalLong.of(catalogStats.sizeInBytes().longValue()); + } + + @Override + public OptionalLong numRows() { + return toJavaOptional(catalogStats.rowCount()) + .map(r -> OptionalLong.of(r.longValue())) + .orElse(OptionalLong.empty()); + } + + @Override + public Map columnStats() { + return colStatsMap; + } + }; + } + + private static Map buildColumnStats( + CatalogStatistics catalogStats, Map columnTypes) { + Map colStats = + scala.collection.JavaConverters.mapAsJavaMapConverter(catalogStats.colStats()).asJava(); + + if (colStats.isEmpty()) { + return Collections.emptyMap(); + } + + Map result = new HashMap<>(); + for (Map.Entry entry : colStats.entrySet()) { + String colName = entry.getKey(); + CatalogColumnStat stat = entry.getValue(); + DataType dataType = columnTypes.get(colName); + + if (dataType == null) { + continue; + } + + NamedReference ref = FieldReference.apply(colName); + int version = stat.version(); + + // Eagerly parse min/max to avoid repeated fromExternalString calls + Optional minValue = + toJavaOptional(stat.min()) + .map( + minStr -> + CatalogColumnStat$.MODULE$.fromExternalString( + minStr, colName, dataType, version)); + Optional maxValue = + toJavaOptional(stat.max()) + .map( + maxStr -> + CatalogColumnStat$.MODULE$.fromExternalString( + maxStr, colName, dataType, version)); + OptionalLong distinctCount = + toJavaOptional(stat.distinctCount()) + .map(d -> OptionalLong.of(d.longValue())) + .orElse(OptionalLong.empty()); + OptionalLong nullCount = + toJavaOptional(stat.nullCount()) + .map(n -> OptionalLong.of(n.longValue())) + .orElse(OptionalLong.empty()); + OptionalLong avgLen = + toJavaOptional(stat.avgLen()) + .map(v -> OptionalLong.of(((Number) v).longValue())) + .orElse(OptionalLong.empty()); + OptionalLong maxLen = + toJavaOptional(stat.maxLen()) + .map(v -> OptionalLong.of(((Number) v).longValue())) + .orElse(OptionalLong.empty()); + + ColumnStatistics v2ColStats = + new ColumnStatistics() { + @Override + public OptionalLong distinctCount() { + return distinctCount; + } + + @Override + public Optional min() { + return minValue; + } + + @Override + public Optional max() { + return maxValue; + } + + @Override + public OptionalLong nullCount() { + return nullCount; + } + + @Override + public OptionalLong avgLen() { + return avgLen; + } + + @Override + public OptionalLong maxLen() { + return maxLen; + } + }; + result.put(ref, v2ColStats); + } + return Collections.unmodifiableMap(result); + } +} diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/DeltaV2TestBase.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/DeltaV2TestBase.java index f07f57fca8f..16dc480014a 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/DeltaV2TestBase.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/DeltaV2TestBase.java @@ -65,16 +65,67 @@ protected void createEmptyTestTable(String path, String tableName) { "CREATE TABLE %s (id INT, name STRING) USING delta LOCATION '%s'", tableName, path)); } + protected void createEmptyPartitionedTestTable(String path, String tableName) { + spark.sql( + String.format( + "CREATE TABLE %s (id INT, name STRING) USING delta PARTITIONED BY (name) LOCATION '%s'", + tableName, path)); + } + protected void createSchemaEvolutionTestTable(String path, String tableName) { spark.sql( String.format( - "CREATE TABLE %s (id INT NOT NULL, name String, value FLOAT) USING delta LOCATION '%s'" + "CREATE TABLE %s (id INT NOT NULL, " + + "name String, value FLOAT, " + + "info STRUCT) USING delta LOCATION '%s'" + "TBLPROPERTIES (" + "'delta.columnMapping.mode' = 'name', " + "'delta.enableTypeWidening' = 'true')", tableName, path)); spark.sql( - String.format("INSERT INTO %s VALUES (1, 'Alice', 10.5), (2,'Bob', NULL)", tableName)); + String.format( + "INSERT INTO %s VALUES " + + "(1, 'Alice', 10.5, named_struct('col1', 27, 'col2', 'LA')), " + + "(2,'Bob', NULL, named_struct('col1', 30, 'col2', 'NYC'))", + tableName)); + } + + /** A runnable that can throw checked exceptions, for use with {@link #withSQLConf}. */ + @FunctionalInterface + protected interface ThrowingRunnable { + void run() throws Exception; + } + + /** + * Runs the given action with a Spark SQL configuration temporarily set, then restores the + * original value afterwards (similar to Scala's {@code withSQLConf}). + */ + protected void withSQLConf(String key, String value, ThrowingRunnable action) throws Exception { + scala.Option original = spark.conf().getOption(key); + spark.conf().set(key, value); + try { + action.run(); + } finally { + if (original.isDefined()) { + spark.conf().set(key, original.get()); + } else { + spark.conf().unset(key); + } + } + } + + /** + * Runs the given action and drops the specified tables afterwards, similar to Scala's {@code + * withTable}. + */ + protected void withTable(String[] tableNames, ThrowingRunnable action) throws Exception { + try { + action.run(); + } finally { + for (String tableName : tableNames) { + spark.sql(String.format("DROP TABLE IF EXISTS %s", tableName)); + } + } } protected static void createPartitionedTable(String tableName, String path) { diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/InternalRowTestUtils.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/InternalRowTestUtils.java index 222908952b4..f24e7ba636d 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/InternalRowTestUtils.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/InternalRowTestUtils.java @@ -26,6 +26,7 @@ import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.catalyst.expressions.GenericInternalRow; import org.apache.spark.sql.execution.datasources.PartitionedFile; +import org.apache.spark.sql.vectorized.ColumnarBatch; import org.apache.spark.unsafe.types.UTF8String; import scala.Function1; import scala.collection.Iterator; @@ -48,6 +49,23 @@ public Iterator apply(PartitionedFile file) { }; } + /** + * Create a mock reader that returns ColumnarBatch objects through Iterator<InternalRow>. + * + *

    Mimics Spark vectorized mode where ColumnarBatch is passed via type erasure. + */ + @SuppressWarnings("unchecked") + public static Function1> mockBatchReader( + List batches) { + return new AbstractFunction1>() { + @Override + public Iterator apply(PartitionedFile file) { + return (Iterator) + (Iterator) CollectionConverters.asScala(batches.iterator()); + } + }; + } + /** Collect all rows from iterator into a list, copying each row. */ public static List collectRows(Iterator iter) { List result = new ArrayList<>(); @@ -57,6 +75,17 @@ public static List collectRows(Iterator iter) { return result; } + /** Collect all ColumnarBatch objects from an iterator (for vectorized mode testing). */ + @SuppressWarnings("unchecked") + public static List collectBatches(Iterator iter) { + Iterator objectIter = (Iterator) (Iterator) iter; + List result = new ArrayList<>(); + while (objectIter.hasNext()) { + result.add((ColumnarBatch) objectIter.next()); + } + return result; + } + /** Create an InternalRow from values. Strings are converted to UTF8String. */ public static InternalRow row(Object... values) { Object[] converted = new Object[values.length]; diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/V2ReadTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/V2ReadTest.java index 81d9d37bd6a..7565169b2b5 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/V2ReadTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/V2ReadTest.java @@ -16,6 +16,7 @@ package io.delta.spark.internal.v2; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.File; @@ -102,4 +103,51 @@ public void testDeletionVectorRead(@TempDir File tempDir) throws Exception { // 500 odd numbers from 0-999: 1, 3, 5, ..., 999 assertTrue(count == 500, "Expected 500 rows after DV filtering, got " + count); } + + @Test + public void testPartitionedJoinEliminatesShuffle(@TempDir File tempDir) { + String tablePath = tempDir.getAbsolutePath(); + + // Create a partitioned Delta table via V1 + spark.sql( + str( + "CREATE TABLE delta.`%s` (id INT, data STRING, part INT) " + + "USING delta PARTITIONED BY (part)", + tablePath)); + spark.sql( + str( + "INSERT INTO delta.`%s` VALUES (1, 'a', 1), (2, 'b', 1), (3, 'c', 2), (4, 'd', 3)", + tablePath)); + + // Disable broadcast join so Spark must use shuffle or partition-aware join. + // Enable V2 bucketing so Spark recognizes KeyGroupedPartitioning from + // SupportsReportPartitioning (default is false in Spark 4.0, true in 4.1). + withSQLConf( + "spark.sql.autoBroadcastJoinThreshold", + "-1", + () -> + withSQLConf( + "spark.sql.sources.v2.bucketing.enabled", + "true", + () -> { + // Self-join on partition column via DSv2 catalog + String joinQuery = + str( + "SELECT a.id, b.data FROM dsv2.delta.`%s` a " + + "JOIN dsv2.delta.`%s` b ON a.part = b.part", + tablePath, tablePath); + + String explainOutput = + spark.sql(joinQuery).queryExecution().executedPlan().toString(); + + // With SupportsReportPartitioning + HasPartitionKey, Spark should recognize + // both sides are KeyGroupedPartitioned on 'part' and skip the shuffle + // (no Exchange node) + assertFalse( + explainOutput.contains("Exchange"), + "Expected no Exchange (shuffle) in plan for partition-aligned join, " + + "but found one.\nPlan:\n" + + explainOutput); + })); + } } diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/V2StreamingReadTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/V2StreamingReadTest.java index ba142f97118..a529b52e507 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/V2StreamingReadTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/V2StreamingReadTest.java @@ -35,8 +35,6 @@ import org.apache.spark.sql.delta.stats.StatisticsCollection; import org.apache.spark.sql.streaming.StreamingQuery; import org.apache.spark.sql.streaming.StreamingQueryException; -import org.apache.spark.sql.types.DataTypes; -import org.apache.spark.sql.types.StructType; import org.junit.jupiter.api.*; import org.junit.jupiter.api.io.TempDir; import scala.Option; @@ -315,22 +313,62 @@ public void testNestedColumnAdditionDetectedOnRestart(@TempDir File deltaTablePa String dsv2TableRef = str("dsv2.delta.`%s`", tablePath); // Create table with struct column: data STRUCT - StructType initialSchema = - DataTypes.createStructType( - Arrays.asList( - DataTypes.createStructField("id", DataTypes.StringType, true), - DataTypes.createStructField( - "data", - DataTypes.createStructType( - Arrays.asList( - DataTypes.createStructField("x", DataTypes.IntegerType, true))), - true))); + spark.sql( + str("CREATE TABLE delta.`%s` (id STRING, data STRUCT) USING delta", tablePath)); + spark.sql(str("INSERT INTO delta.`%s` VALUES ('0', named_struct('x', 1))", tablePath)); - spark - .createDataFrame(Arrays.asList(RowFactory.create("0", RowFactory.create(1))), initialSchema) - .write() - .format("delta") - .save(tablePath); + // Start streaming and process initial data + File checkpointDir = new File(deltaTablePath, "_checkpoint"); + Dataset streamingDF = spark.readStream().table(dsv2TableRef); + StreamingQuery query = + streamingDF + .writeStream() + .format("noop") + .option("checkpointLocation", checkpointDir.getAbsolutePath()) + .start(); + query.processAllAvailable(); + query.stop(); + + // Evolve struct via ALTER TABLE (metadata-only, no file deletion): + // add nested field y -> data STRUCT + spark.sql(str("ALTER TABLE delta.`%s` ADD COLUMNS (data.y INT)", tablePath)); + + // Restart with stale DataFrame — should fail with schema mismatch + StreamingQueryException ex = + assertThrows( + StreamingQueryException.class, + () -> { + StreamingQuery q = + streamingDF + .writeStream() + .format("noop") + .option("checkpointLocation", checkpointDir.getAbsolutePath()) + .start(); + try { + q.processAllAvailable(); + } finally { + q.stop(); + } + }); + assertInstanceOf(DeltaIllegalStateException.class, ex.cause()); + assertTrue( + ex.getMessage().contains("DELTA_STREAMING_SCHEMA_MISMATCH_ON_RESTART"), + "Expected DELTA_STREAMING_SCHEMA_MISMATCH_ON_RESTART but got: " + ex.getMessage()); + } + + // TODO(#6232): v2 source cannot adopt type widening schema change without refreshing the + // dataframe due to the lack of support in spark stream engine. Throw an error at stream + // start time to instruct user. + @Test + public void testNestedTypeWideningDetectedOnRestart(@TempDir File deltaTablePath) + throws Exception { + String tablePath = deltaTablePath.getAbsolutePath(); + String dsv2TableRef = str("dsv2.delta.`%s`", tablePath); + + // Create table with struct column: data STRUCT + spark.sql( + str("CREATE TABLE delta.`%s` (id STRING, data STRUCT) USING delta", tablePath)); + spark.sql(str("INSERT INTO delta.`%s` VALUES ('0', named_struct('x', 1))", tablePath)); // Start streaming and process initial data File checkpointDir = new File(deltaTablePath, "_checkpoint"); @@ -344,26 +382,70 @@ public void testNestedColumnAdditionDetectedOnRestart(@TempDir File deltaTablePa query.processAllAvailable(); query.stop(); - // Evolve struct: add nested field y -> data STRUCT - StructType evolvedSchema = - DataTypes.createStructType( - Arrays.asList( - DataTypes.createStructField("id", DataTypes.StringType, true), - DataTypes.createStructField( - "data", - DataTypes.createStructType( - Arrays.asList( - DataTypes.createStructField("x", DataTypes.IntegerType, true), - DataTypes.createStructField("y", DataTypes.IntegerType, true))), - true))); - spark - .createDataFrame( - Arrays.asList(RowFactory.create("1", RowFactory.create(2, 3))), evolvedSchema) - .write() - .format("delta") - .mode("overwrite") - .option("overwriteSchema", "true") - .save(tablePath); + // Widen nested field type via ALTER TABLE (metadata-only, no file deletion): + // data.x INT -> data.x BIGINT + spark.sql( + str( + "ALTER TABLE delta.`%s` SET TBLPROPERTIES ('delta.enableTypeWidening' = 'true')", + tablePath)); + spark.sql(str("ALTER TABLE delta.`%s` ALTER COLUMN data.x TYPE BIGINT", tablePath)); + + // Restart with stale DataFrame — should fail with schema mismatch + StreamingQueryException ex = + assertThrows( + StreamingQueryException.class, + () -> { + StreamingQuery q = + streamingDF + .writeStream() + .format("noop") + .option("checkpointLocation", checkpointDir.getAbsolutePath()) + .start(); + try { + q.processAllAvailable(); + } finally { + q.stop(); + } + }); + assertInstanceOf(DeltaIllegalStateException.class, ex.cause()); + assertTrue( + ex.getMessage().contains("DELTA_STREAMING_SCHEMA_MISMATCH_ON_RESTART"), + "Expected DELTA_STREAMING_SCHEMA_MISMATCH_ON_RESTART but got: " + ex.getMessage()); + } + + // TODO(#6232): v2 source cannot adopt type widening schema change without refreshing the + // dataframe due to the lack of support in spark stream engine. Throw an error at stream + // start time to instruct user. + @Test + public void testNestedNullabilityRelaxDetectedOnRestart(@TempDir File deltaTablePath) + throws Exception { + String tablePath = deltaTablePath.getAbsolutePath(); + String dsv2TableRef = str("dsv2.delta.`%s`", tablePath); + + // Create table via SQL DDL to preserve the NOT NULL constraint on the nested field. + // DataFrame writes go through ImplicitMetadataOperation which calls schema.asNullable, + // forcing all fields (including nested ones) to nullable — losing the NOT NULL. + spark.sql( + str( + "CREATE TABLE delta.`%s` (id STRING, data STRUCT) USING delta", + tablePath)); + spark.sql(str("INSERT INTO delta.`%s` VALUES ('0', named_struct('x', 1))", tablePath)); + + // Start streaming and process initial data + File checkpointDir = new File(deltaTablePath, "_checkpoint"); + Dataset streamingDF = spark.readStream().table(dsv2TableRef); + StreamingQuery query = + streamingDF + .writeStream() + .format("noop") + .option("checkpointLocation", checkpointDir.getAbsolutePath()) + .start(); + query.processAllAvailable(); + query.stop(); + + // Relax nullability via ALTER TABLE (metadata-only, no file deletion): + // data.x NOT NULL -> data.x nullable + spark.sql(str("ALTER TABLE delta.`%s` ALTER COLUMN data.x DROP NOT NULL", tablePath)); // Restart with stale DataFrame — should fail with schema mismatch StreamingQueryException ex = diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/ddl/DDLRequestContextTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/ddl/DDLRequestContextTest.java new file mode 100644 index 00000000000..f3923c86e92 --- /dev/null +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/ddl/DDLRequestContextTest.java @@ -0,0 +1,112 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.spark.internal.v2.ddl; + +import static org.junit.jupiter.api.Assertions.*; + +import io.delta.kernel.defaults.engine.DefaultEngine; +import io.delta.kernel.engine.Engine; +import io.delta.kernel.expressions.Column; +import io.delta.kernel.transaction.DataLayoutSpec; +import io.delta.kernel.types.IntegerType; +import io.delta.kernel.types.StringType; +import io.delta.kernel.types.StructField; +import io.delta.kernel.types.StructType; +import io.delta.kernel.types.TimestampType; +import io.delta.spark.internal.v2.snapshot.unitycatalog.UCTableInfo; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.apache.hadoop.conf.Configuration; +import org.apache.spark.sql.connector.catalog.Identifier; +import org.junit.jupiter.api.Test; + +/** Unit tests for the {@link DDLRequestContext} POJO. */ +public class DDLRequestContextTest { + + private static final Engine ENGINE = DefaultEngine.create(new Configuration()); + + @Test + public void testConstructionWithAllFields() { + Identifier ident = Identifier.of(new String[] {"prod_catalog", "analytics"}, "page_views"); + StructType schema = + new StructType() + .add(new StructField("user_id", IntegerType.INTEGER, false)) + .add(new StructField("url", StringType.STRING, false)) + .add(new StructField("event_ts", TimestampType.TIMESTAMP, false)); + DataLayoutSpec clustering = + DataLayoutSpec.clustered(List.of(new Column("user_id"), new Column("event_ts"))); + UCTableInfo ucInfo = + new UCTableInfo( + "uc-table-id-123", + "/managed/prod_catalog/analytics/page_views", + "https://uc.example.com", + Map.of("token", "fake-token")); + + DDLRequestContext request = + new DDLRequestContext( + ident, + "/managed/prod_catalog/analytics/page_views", + schema, + Map.of("delta.appendOnly", "true", "delta.logRetentionDuration", "interval 30 days"), + Optional.of("Tracks page view events from the analytics pipeline"), + clustering, + ENGINE, + Optional.of(ucInfo), + "Delta-Spark-DSv2/4.0.0"); + + assertEquals("page_views", request.ident().name()); + assertArrayEquals(new String[] {"prod_catalog", "analytics"}, request.ident().namespace()); + assertEquals("/managed/prod_catalog/analytics/page_views", request.tablePath()); + assertEquals(3, request.kernelSchema().length()); + assertEquals("true", request.properties().get("delta.appendOnly")); + assertEquals( + Optional.of("Tracks page view events from the analytics pipeline"), request.comment()); + assertTrue(request.dataLayoutSpec().hasClustering()); + assertEquals(2, request.dataLayoutSpec().getClusteringColumns().size()); + assertTrue(request.ucTableInfo().isPresent()); + assertEquals("uc-table-id-123", request.ucTableInfo().get().getTableId()); + assertEquals("Delta-Spark-DSv2/4.0.0", request.engineInfo()); + } + + @Test + public void testPropertiesAreImmutableAndDefensivelyCopied() { + Identifier ident = Identifier.of(new String[] {"db"}, "tbl"); + StructType schema = new StructType().add(new StructField("id", IntegerType.INTEGER, false)); + + HashMap original = new HashMap<>(); + original.put("k", "v"); + DDLRequestContext request = + new DDLRequestContext( + ident, + "/p", + schema, + original, + /* comment = */ Optional.empty(), + DataLayoutSpec.noDataLayout(), + ENGINE, + /* ucTableInfo = */ Optional.empty(), + "e"); + + // Mutating the original map must not leak into the DDLRequestContext + original.put("injected", "bad"); + assertNull(request.properties().get("injected")); + + // The returned map must be unmodifiable + assertThrows(UnsupportedOperationException.class, () -> request.properties().put("x", "y")); + } +} diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkGoldenTableTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkGoldenTableTest.java index fda6c0f4003..48f4fc5886e 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkGoldenTableTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkGoldenTableTest.java @@ -194,7 +194,7 @@ public void testDsv2Internal() throws Exception { // input filters new Filter[] { new GreaterThan("cnt", 10), // supported data filter - new StringStartsWith("name", "foo"), // unsupported data filter + new StringStartsWith("name", "foo"), // supported data filter new EqualTo("date", "2025-09-01"), // supported partition filter new StringEndsWith("city", "York"), // unsupported partition filter }, @@ -205,10 +205,15 @@ public void testDsv2Internal() throws Exception { new StringEndsWith("city", "York"), }, // expected pushed filters - new Filter[] {new GreaterThan("cnt", 10), new EqualTo("date", "2025-09-01")}, + new Filter[] { + new GreaterThan("cnt", 10), + new StringStartsWith("name", "foo"), + new EqualTo("date", "2025-09-01") + }, // expected pushed kernel predicates new Predicate[] { new Predicate(">", new Column("cnt"), Literal.ofInt(10)), + new Predicate("STARTS_WITH", new Column("name"), Literal.ofString("foo")), new Predicate("=", new Column("date"), Literal.ofString("2025-09-01")) }, // expected data filters @@ -217,7 +222,10 @@ public void testDsv2Internal() throws Exception { Optional.of( new Predicate( "AND", - new Predicate(">", new Column("cnt"), Literal.ofInt(10)), + new Predicate( + "AND", + new Predicate(">", new Column("cnt"), Literal.ofInt(10)), + new Predicate("STARTS_WITH", new Column("name"), Literal.ofString("foo"))), new Predicate("=", new Column("date"), Literal.ofString("2025-09-01"))))); // case 2: OR and NOT filters @@ -240,11 +248,17 @@ public void testDsv2Internal() throws Exception { }, // expected pushed filters new Filter[] { + new Or(new GreaterThan("cnt", 10), new StringStartsWith("name", "foo")), new Or(new EqualTo("cnt", 50), new EqualTo("date", "2025-10-01")), new Not(new And(new GreaterThan("cnt", 100), new EqualTo("date", "2025-09-01"))), + new Not(new Or(new EqualTo("name", "foo"), new StringStartsWith("city", "New"))) }, // expected pushed kernel predicates new Predicate[] { + new Predicate( + "OR", + new Predicate(">", new Column("cnt"), Literal.ofInt(10)), + new Predicate("STARTS_WITH", new Column("name"), Literal.ofString("foo"))), new Predicate( "OR", new Predicate("=", new Column("cnt"), Literal.ofInt(50)), @@ -254,7 +268,13 @@ public void testDsv2Internal() throws Exception { new Predicate( "AND", new Predicate(">", new Column("cnt"), Literal.ofInt(100)), - new Predicate("=", new Column("date"), Literal.ofString("2025-09-01")))) + new Predicate("=", new Column("date"), Literal.ofString("2025-09-01")))), + new Predicate( + "NOT", + new Predicate( + "OR", + new Predicate("=", new Column("name"), Literal.ofString("foo")), + new Predicate("STARTS_WITH", new Column("city"), Literal.ofString("New")))) }, // expected data filters new Filter[] { @@ -264,19 +284,39 @@ public void testDsv2Internal() throws Exception { new Not(new Or(new EqualTo("name", "foo"), new StringStartsWith("city", "New"))) }, // expected kernel scan builder predicate + // reduce(And::new) over 4 predicates gives left-associative nesting: + // AND(AND(AND(pred1, pred2), pred3), pred4) Optional.of( new Predicate( "AND", new Predicate( - "OR", - new Predicate("=", new Column("cnt"), Literal.ofInt(50)), - new Predicate("=", new Column("date"), Literal.ofString("2025-10-01"))), + "AND", + new Predicate( + "AND", + new Predicate( + "OR", + new Predicate(">", new Column("cnt"), Literal.ofInt(10)), + new Predicate( + "STARTS_WITH", new Column("name"), Literal.ofString("foo"))), + new Predicate( + "OR", + new Predicate("=", new Column("cnt"), Literal.ofInt(50)), + new Predicate( + "=", new Column("date"), Literal.ofString("2025-10-01")))), + new Predicate( + "NOT", + new Predicate( + "AND", + new Predicate(">", new Column("cnt"), Literal.ofInt(100)), + new Predicate( + "=", new Column("date"), Literal.ofString("2025-09-01"))))), new Predicate( "NOT", new Predicate( - "AND", - new Predicate(">", new Column("cnt"), Literal.ofInt(100)), - new Predicate("=", new Column("date"), Literal.ofString("2025-09-01"))))))); + "OR", + new Predicate("=", new Column("name"), Literal.ofString("foo")), + new Predicate( + "STARTS_WITH", new Column("city"), Literal.ofString("New"))))))); // check SupportsRuntimeV2Filtering // city = 'hz' AND date = '20180520' diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkMicroBatchStreamTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkMicroBatchStreamTest.java index 6e96bbb17d8..e86291de0bc 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkMicroBatchStreamTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkMicroBatchStreamTest.java @@ -17,8 +17,9 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.*; -import static org.junit.jupiter.api.Assumptions.assumeFalse; +import io.delta.kernel.CommitActions; +import io.delta.kernel.data.ColumnarBatch; import io.delta.kernel.utils.CloseableIterator; import io.delta.spark.internal.v2.DeltaV2TestBase; import io.delta.spark.internal.v2.snapshot.PathBasedSnapshotManager; @@ -28,10 +29,13 @@ import java.sql.Timestamp; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -54,6 +58,7 @@ import org.apache.spark.sql.delta.sources.ReadMaxBytes; import org.apache.spark.sql.delta.storage.ClosableIterator; import org.apache.spark.sql.delta.util.JsonUtils; +import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -942,7 +947,7 @@ private static Stream removeFileScenarios() { String sourceTableName = "merge_source_" + System.nanoTime(); sql( "CREATE TABLE %s (id INT, name STRING) USING delta LOCATION '%s'", - sourceTableName, tempDir.getAbsolutePath() + "_source"); + sourceTableName, new File(tempDir, "source").getAbsolutePath()); sql("INSERT INTO %s VALUES (2, 'UpdatedUser2'), (4, 'User4')", sourceTableName); // Perform MERGE operation @@ -954,7 +959,400 @@ private static Stream removeFileScenarios() { sql("DROP TABLE IF EXISTS %s", sourceTableName); }, - "MERGE: Matched (REMOVE+ADD) and not matched (ADD)")); + "MERGE: Matched (REMOVE+ADD) and not matched (ADD)"), + + // Full table delete (only RemoveFiles, no AddFiles) + Arguments.of( + (ScenarioSetup) + (tableName, tempDir) -> { + sql("INSERT INTO %s VALUES (1, 'User1'), (2, 'User2')", tableName); + sql("DELETE FROM %s", tableName); + }, + "DELETE: Full table delete (removes only, no adds)")); + } + + // ================================================================================================ + // Shared helpers for ignoreDeletes / skipChangeCommits / ignoreChanges parity tests + // ================================================================================================ + + /** + * Runs a parity test: verifies that DSv1 and DSv2 produce identical file changes for the given + * option and scenario. + */ + private void runFileChangeParityTest( + ScenarioSetup scenarioSetup, + boolean isInitialSnapshot, + String testDescription, + File tempDir, + String optionName, + boolean usePartitionedTable) + throws Exception { + String testTablePath = tempDir.getAbsolutePath(); + String testTableName = + "test_" + optionName + "_" + Math.abs(testDescription.hashCode()) + "_" + System.nanoTime(); + if (usePartitionedTable) { + createEmptyPartitionedTestTable(testTablePath, testTableName); + } else { + createEmptyTestTable(testTablePath, testTableName); + } + + scenarioSetup.setup(testTableName, tempDir); + + long fromVersion = 0L; + long fromIndex = DeltaSourceOffset.BASE_INDEX(); + DeltaOptions options = createDeltaOptions(optionName, "true"); + + // DSv1 + DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); + DeltaSource deltaSource = createDeltaSource(deltaLog, testTablePath, options); + Option endOffset = Option.empty(); + ClosableIterator deltaChanges = + deltaSource.getFileChanges( + fromVersion, fromIndex, isInitialSnapshot, endOffset, /* verifyMetadataAction= */ true); + List deltaFilesList = new ArrayList<>(); + while (deltaChanges.hasNext()) { + deltaFilesList.add(deltaChanges.next()); + } + deltaChanges.close(); + + // DSv2 + Configuration hadoopConf = new Configuration(); + PathBasedSnapshotManager snapshotManager = + new PathBasedSnapshotManager(testTablePath, hadoopConf); + SparkMicroBatchStream stream = + createTestStreamWithDefaults(snapshotManager, hadoopConf, options); + try (CloseableIterator kernelChanges = + stream.getFileChanges(fromVersion, fromIndex, isInitialSnapshot, Optional.empty())) { + List kernelFilesList = new ArrayList<>(); + while (kernelChanges.hasNext()) { + kernelFilesList.add(kernelChanges.next()); + } + compareFileChanges(deltaFilesList, kernelFilesList); + } + } + + /** + * Runs a throws-parity test: verifies that both DSv1 and DSv2 throw UnsupportedOperationException + * after the same number of next() calls. + */ + private void runFileChangeThrowsParityTest( + ScenarioSetup scenarioSetup, + boolean isInitialSnapshot, + String testDescription, + File tempDir, + String optionName) + throws Exception { + String testTablePath = tempDir.getAbsolutePath(); + String testTableName = + "test_" + optionName + "_" + Math.abs(testDescription.hashCode()) + "_" + System.nanoTime(); + createEmptyTestTable(testTablePath, testTableName); + + scenarioSetup.setup(testTableName, tempDir); + + long fromVersion = 0L; + long fromIndex = DeltaSourceOffset.BASE_INDEX(); + Option endOffset = Option.empty(); + DeltaOptions options = createDeltaOptions(optionName, "true"); + + // DSv1 + DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); + DeltaSource deltaSource = createDeltaSource(deltaLog, testTablePath, options); + + AtomicInteger dsv1SuccessfulCalls = new AtomicInteger(0); + assertThrows( + UnsupportedOperationException.class, + () -> { + ClosableIterator deltaChanges = + deltaSource.getFileChanges( + fromVersion, + fromIndex, + isInitialSnapshot, + endOffset, + /* verifyMetadataAction= */ true); + try { + while (deltaChanges.hasNext()) { + deltaChanges.next(); + dsv1SuccessfulCalls.incrementAndGet(); + } + } finally { + deltaChanges.close(); + } + }, + String.format( + "DSv1 should throw on change commit with %s for: %s", optionName, testDescription)); + + // DSv2 + Configuration hadoopConf = new Configuration(); + PathBasedSnapshotManager snapshotManager = + new PathBasedSnapshotManager(testTablePath, hadoopConf); + SparkMicroBatchStream stream = + createTestStreamWithDefaults(snapshotManager, hadoopConf, options); + + AtomicInteger dsv2SuccessfulCalls = new AtomicInteger(0); + assertThrows( + UnsupportedOperationException.class, + () -> { + CloseableIterator kernelChanges = + stream.getFileChanges(fromVersion, fromIndex, isInitialSnapshot, Optional.empty()); + try { + while (kernelChanges.hasNext()) { + kernelChanges.next(); + dsv2SuccessfulCalls.incrementAndGet(); + } + } finally { + kernelChanges.close(); + } + }, + String.format( + "DSv2 should throw on change commit with %s for: %s", optionName, testDescription)); + + assertEquals( + dsv1SuccessfulCalls.get(), + dsv2SuccessfulCalls.get(), + String.format( + "DSv1 and DSv2 should throw after the same number of next() calls for: %s. " + + "DSv1=%d, DSv2=%d", + testDescription, dsv1SuccessfulCalls.get(), dsv2SuccessfulCalls.get())); + } + + // ================================================================================================ + // Tests for ignoreDeletes parity between DSv1 and DSv2 + // ================================================================================================ + + /** + * Verifies that with ignoreDeletes=true, both DSv1 and DSv2 produce the same file changes for + * delete-only commits (commits with only RemoveFile actions, no AddFile actions). The delete-only + * commit should be silently skipped (only sentinels emitted, no data files). + */ + @ParameterizedTest + @MethodSource("deleteOnlyScenarios") + public void testGetFileChanges_withIgnoreDeletes_deleteOnlyParity( + ScenarioSetup scenarioSetup, + boolean isInitialSnapshot, + String testDescription, + @TempDir File tempDir) + throws Exception { + runFileChangeParityTest( + scenarioSetup, isInitialSnapshot, testDescription, tempDir, "ignoreDeletes", true); + } + + /** + * Verifies that with ignoreDeletes=true, both DSv1 and DSv2 still throw on commits containing + * both adds and removes (e.g., UPDATE, MERGE), since ignoreDeletes only suppresses delete-only + * commits. + */ + @ParameterizedTest + @MethodSource("changeCommitScenarios") + public void testGetFileChanges_withIgnoreDeletes_changeCommitStillThrows( + ScenarioSetup scenarioSetup, + boolean isInitialSnapshot, + String testDescription, + @TempDir File tempDir) + throws Exception { + runFileChangeThrowsParityTest( + scenarioSetup, isInitialSnapshot, testDescription, tempDir, "ignoreDeletes"); + } + + // ================================================================================================ + // Tests for skipChangeCommits parity between DSv1 and DSv2 + // ================================================================================================ + + /** + * Verifies that with skipChangeCommits=true, both DSv1 and DSv2 produce the same file changes for + * delete-only commits. Since skipChangeCommits suppresses all commits containing RemoveFile + * actions, these commits should be silently skipped (only sentinels emitted, no data files). + */ + @ParameterizedTest + @MethodSource("deleteOnlyScenarios") + public void testGetFileChanges_withSkipChangeCommits_deleteOnlyParity( + ScenarioSetup scenarioSetup, + boolean isInitialSnapshot, + String testDescription, + @TempDir File tempDir) + throws Exception { + runFileChangeParityTest( + scenarioSetup, isInitialSnapshot, testDescription, tempDir, "skipChangeCommits", true); + } + + /** + * Verifies that with skipChangeCommits=true, both DSv1 and DSv2 silently skip commits containing + * both adds and removes (e.g., UPDATE, MERGE), instead of throwing. This is the key behavioral + * difference from ignoreDeletes, which throws on such commits. + */ + @ParameterizedTest + @MethodSource("changeCommitScenarios") + public void testGetFileChanges_withSkipChangeCommits_changeCommitParity( + ScenarioSetup scenarioSetup, + boolean isInitialSnapshot, + String testDescription, + @TempDir File tempDir) + throws Exception { + runFileChangeParityTest( + scenarioSetup, isInitialSnapshot, testDescription, tempDir, "skipChangeCommits", false); + } + + // ================================================================================================ + // Tests for ignoreChanges parity between DSv1 and DSv2 + // ================================================================================================ + + /** + * Verifies that with ignoreChanges=true, both DSv1 and DSv2 produce the same file changes for + * delete-only commits. Since ignoreChanges implies shouldAllowDeletes, these commits should be + * silently skipped (only sentinels emitted, no data files). + */ + @ParameterizedTest + @MethodSource("deleteOnlyScenarios") + public void testGetFileChanges_withIgnoreChanges_deleteOnlyParity( + ScenarioSetup scenarioSetup, + boolean isInitialSnapshot, + String testDescription, + @TempDir File tempDir) + throws Exception { + runFileChangeParityTest( + scenarioSetup, isInitialSnapshot, testDescription, tempDir, "ignoreChanges", true); + } + + /** + * Verifies that with ignoreChanges=true, both DSv1 and DSv2 produce the same file changes for + * change commits (commits containing both AddFile and RemoveFile actions). Unlike ignoreDeletes, + * ignoreChanges allows these commits through and emits their AddFiles. + */ + @ParameterizedTest + @MethodSource("changeCommitScenarios") + public void testGetFileChanges_withIgnoreChanges_changeCommitParity( + ScenarioSetup scenarioSetup, + boolean isInitialSnapshot, + String testDescription, + @TempDir File tempDir) + throws Exception { + runFileChangeParityTest( + scenarioSetup, isInitialSnapshot, testDescription, tempDir, "ignoreChanges", false); + } + + // TODO(#5319): test the combinations of ignoreDeletes, skipChangeCommits, and ignoreChanges + // ================================================================================================ + // Shared scenario providers for ignoreDeletes, skipChangeCommits, and ignoreChanges tests + // ================================================================================================ + + /** + * Provides delete-only scenarios: commits with only RemoveFile actions and no AddFile actions. + * Used by ignoreDeletes, skipChangeCommits, and ignoreChanges tests. + * + *

    Arguments: (ScenarioSetup, isInitialSnapshot, testDescription) + */ + private static Stream deleteOnlyScenarios() { + return Stream.of( + Arguments.of( + (ScenarioSetup) + (tableName, tempDir) -> { + sql("INSERT INTO %s VALUES (1, 'User1'), (2, 'User2')", tableName); + sql("INSERT INTO %s VALUES (3, 'User3'), (4, 'User4')", tableName); + sql("DELETE FROM %s", tableName); + }, + false, + "Full table delete"), + Arguments.of( + (ScenarioSetup) + (tableName, tempDir) -> { + sql("INSERT INTO %s VALUES (1, 'User1'), (2, 'User2')", tableName); + sql("DELETE FROM %s", tableName); + sql("INSERT INTO %s VALUES (3, 'User3'), (4, 'User4')", tableName); + }, + false, + "Insert-Delete-Insert: data resumes after delete"), + Arguments.of( + (ScenarioSetup) + (tableName, tempDir) -> { + sql("INSERT INTO %s VALUES (1, 'PartA'), (2, 'PartA'), (3, 'PartB')", tableName); + sql("DELETE FROM %s WHERE name = 'PartA'", tableName); + }, + false, + "Partitioned table: delete entire partition"), + Arguments.of( + (ScenarioSetup) + (tableName, tempDir) -> { + sql( + "ALTER TABLE %s SET TBLPROPERTIES ('delta.enableDeletionVectors' = true)", + tableName); + sql( + "INSERT INTO %s SELECT /*+ COALESCE(1) */ * FROM VALUES " + + "(1, 'User1'), (2, 'User2'), (3, 'User3') AS t(id, name)", + tableName); + sql("DELETE FROM %s WHERE id >= 1", tableName); + }, + false, + "Full DELETE with DV: full file delete via WHERE clause"), + Arguments.of( + (ScenarioSetup) + (tableName, tempDir) -> { + sql("INSERT INTO %s VALUES (1, 'User1'), (2, 'User2')", tableName); + sql("INSERT INTO %s VALUES (3, 'User3'), (4, 'User4')", tableName); + sql("DELETE FROM %s", tableName); + }, + true, + "Full table delete with initial snapshot")); + } + + /** + * Provides change-commit scenarios: commits containing both AddFile and RemoveFile actions (e.g., + * UPDATE, MERGE). Used by ignoreDeletes, skipChangeCommits, and ignoreChanges tests — + * ignoreDeletes expects these to throw, skipChangeCommits expects them to be silently skipped, + * and ignoreChanges expects them to pass through with AddFiles emitted. + * + *

    Arguments: (ScenarioSetup, isInitialSnapshot, testDescription) + */ + private static Stream changeCommitScenarios() { + return Stream.of( + Arguments.of( + (ScenarioSetup) + (tableName, tempDir) -> { + sql("INSERT INTO %s VALUES (1, 'User1'), (2, 'User2'), (3, 'User3')", tableName); + sql("UPDATE %s SET name = 'Updated' WHERE id = 1", tableName); + }, + false, + "UPDATE: AddFile + RemoveFile"), + Arguments.of( + (ScenarioSetup) + (tableName, tempDir) -> { + sql("INSERT INTO %s VALUES (1, 'User1'), (2, 'User2'), (3, 'User3')", tableName); + String sourceTableName = "merge_src_" + System.nanoTime(); + sql( + "CREATE TABLE %s (id INT, name STRING) USING delta LOCATION '%s'", + sourceTableName, new File(tempDir, "source").getAbsolutePath()); + sql("INSERT INTO %s VALUES (2, 'UpdatedUser2'), (4, 'User4')", sourceTableName); + sql( + "MERGE INTO %s AS target USING %s AS source ON target.id = source.id " + + "WHEN MATCHED THEN UPDATE SET target.name = source.name " + + "WHEN NOT MATCHED THEN INSERT (id, name) " + + "VALUES (source.id, source.name)", + tableName, sourceTableName); + sql("DROP TABLE IF EXISTS %s", sourceTableName); + }, + false, + "MERGE: AddFile + RemoveFile"), + Arguments.of( + (ScenarioSetup) + (tableName, tempDir) -> { + sql( + "ALTER TABLE %s SET TBLPROPERTIES ('delta.enableDeletionVectors' = true)", + tableName); + // Coalesce to to ensure DV is partial delete + sql( + "INSERT INTO %s SELECT /*+ COALESCE(1) */ * FROM VALUES " + + "(1, 'User1'), (2, 'User2'), (3, 'User3') AS t(id, name)", + tableName); + sql("DELETE FROM %s WHERE id = 1", tableName); + }, + false, + "Partial DELETE with DV: AddFile(with DV) + RemoveFile"), + Arguments.of( + (ScenarioSetup) + (tableName, tempDir) -> { + sql("INSERT INTO %s VALUES (1, 'User1'), (2, 'User2'), (3, 'User3')", tableName); + sql("UPDATE %s SET name = 'Updated' WHERE id = 1", tableName); + }, + true, + "UPDATE with initial snapshot: AddFile + RemoveFile")); } @Test @@ -1648,6 +2046,208 @@ private static Stream planInputPartitionsParameters() { "Same version (3 to 3)")); } + // ================================================================================================ + // Tests for planInputPartitions with excludeRegex + // ================================================================================================ + + /** + * Parameterized test that verifies planInputPartitions correctly applies the excludeRegex read + * option. + * + *

    Uses a partitioned table so file paths contain partition directory names (e.g. + * "category=alpha/..."), making it straightforward to craft regex patterns that target specific + * subsets of files. + * + *

    Verifies correctness by extracting the {@code id} column (always at ordinal 0) from the + * reader output and comparing the resulting set of IDs against the expected set. + */ + @ParameterizedTest + @MethodSource("excludeRegexPlanInputPartitionsParameters") + public void testPlanInputPartitions_excludeRegex( + String excludeRegexPattern, + long fromVersion, + long toVersion, + int[] expectedIds, + boolean isInitialSnapshot, + String testDescription, + @TempDir File tempDir) + throws Exception { + String testTablePath = tempDir.getAbsolutePath(); + String testTableName = + "test_exclude_regex_" + Math.abs(testDescription.hashCode()) + "_" + System.nanoTime(); + + // Create table partitioned by category with 4 versions of data. + // Version 0: CREATE TABLE (empty) + // Version 1: ids {1,2} in category=alpha + // Version 2: ids {3,4} in category=beta + // Version 3: ids {5,6} in category=alpha + // Version 4: ids {7,8} in category=gamma + sql( + "CREATE TABLE %s (id INT, name STRING, category STRING) " + + "USING delta LOCATION '%s' PARTITIONED BY (category)", + testTableName, testTablePath); + sql("INSERT INTO %s VALUES (1, 'Alice', 'alpha'), (2, 'Bob', 'alpha')", testTableName); + sql("INSERT INTO %s VALUES (3, 'Charlie', 'beta'), (4, 'David', 'beta')", testTableName); + sql("INSERT INTO %s VALUES (5, 'Eve', 'alpha'), (6, 'Frank', 'alpha')", testTableName); + sql("INSERT INTO %s VALUES (7, 'Grace', 'gamma'), (8, 'Harry', 'gamma')", testTableName); + + DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); + DeltaSourceOffset startOffset = + new DeltaSourceOffset( + deltaLog.tableId(), fromVersion, DeltaSourceOffset.BASE_INDEX(), isInitialSnapshot); + DeltaSourceOffset endOffset = + new DeltaSourceOffset( + deltaLog.tableId(), toVersion, DeltaSourceOffset.END_INDEX(), isInitialSnapshot); + + DeltaOptions options = createDeltaOptions("excludeRegex", excludeRegexPattern); + + // DSv2: planInputPartitions + createReaderFactory + PathBasedSnapshotManager snapshotManager = + new PathBasedSnapshotManager(testTablePath, spark.sessionState().newHadoopConf()); + org.apache.spark.sql.delta.Snapshot deltaSnapshot = deltaLog.unsafeVolatileSnapshot(); + StructType fullSchema = deltaSnapshot.metadata().schema(); + StructType partitionSchema = deltaSnapshot.metadata().partitionSchema(); + Set partitionColNames = + Arrays.stream(partitionSchema.fieldNames()).collect(Collectors.toSet()); + StructType dataSchema = + new StructType( + Arrays.stream(fullSchema.fields()) + .filter(f -> !partitionColNames.contains(f.name())) + .toArray(StructField[]::new)); + + SparkMicroBatchStream stream = + new SparkMicroBatchStream( + snapshotManager, + snapshotManager.loadLatestSnapshot(), + spark.sessionState().newHadoopConf(), + spark, + options, + testTablePath, + dataSchema, + partitionSchema, + fullSchema, + new org.apache.spark.sql.sources.Filter[0], + Map$.MODULE$.empty()); + + InputPartition[] partitions = stream.planInputPartitions(startOffset, endOffset); + PartitionReaderFactory readerFactory = stream.createReaderFactory(); + + // Extract IDs (column ordinal 0) from reader output. + // We read only the id to avoid schema complications with partitioned tables where + // the vectorized reader may not include partition columns in the InternalRow. + List dsv2Ids = new ArrayList<>(); + for (InputPartition partition : partitions) { + if (readerFactory.supportColumnarReads(partition)) { + PartitionReader reader = + readerFactory.createColumnarReader(partition); + while (reader.next()) { + org.apache.spark.sql.vectorized.ColumnarBatch batch = reader.get(); + for (int rowId = 0; rowId < batch.numRows(); rowId++) { + dsv2Ids.add(batch.getRow(rowId).getInt(0)); + } + } + reader.close(); + } else { + PartitionReader reader = readerFactory.createReader(partition); + while (reader.next()) { + dsv2Ids.add(reader.get().getInt(0)); + } + reader.close(); + } + } + + List expected = + Arrays.stream(expectedIds).sorted().boxed().collect(Collectors.toList()); + Collections.sort(dsv2Ids); + + assertEquals( + expected, + dsv2Ids, + String.format("[%s] ID mismatch: expected=%s, got=%s", testDescription, expected, dsv2Ids)); + } + + /** Provides test parameters for the excludeRegex planInputPartitions test. */ + private static Stream excludeRegexPlanInputPartitionsParameters() { + return Stream.of( + Arguments.of( + /* excludeRegexPattern= */ (String) null, + /* fromVersion= */ 1L, + /* toVersion= */ 4L, + /* expectedIds= */ new int[] {1, 2, 3, 4, 5, 6, 7, 8}, + /* isInitialSnapshot= */ false, + "No excludeRegex - all versions"), + Arguments.of( + /* excludeRegexPattern= */ "category=alpha", + /* fromVersion= */ 1L, + /* toVersion= */ 4L, + /* expectedIds= */ new int[] {3, 4, 7, 8}, + /* isInitialSnapshot= */ false, + "Exclude alpha category"), + Arguments.of( + /* excludeRegexPattern= */ "category=beta", + /* fromVersion= */ 1L, + /* toVersion= */ 4L, + /* expectedIds= */ new int[] {1, 2, 5, 6, 7, 8}, + /* isInitialSnapshot= */ false, + "Exclude beta category"), + Arguments.of( + /* excludeRegexPattern= */ "category=gamma", + /* fromVersion= */ 1L, + /* toVersion= */ 4L, + /* expectedIds= */ new int[] {1, 2, 3, 4, 5, 6}, + /* isInitialSnapshot= */ false, + "Exclude gamma category"), + Arguments.of( + /* excludeRegexPattern= */ "category=(alpha|beta)", + /* fromVersion= */ 1L, + /* toVersion= */ 4L, + /* expectedIds= */ new int[] {7, 8}, + /* isInitialSnapshot= */ false, + "Exclude alpha and beta categories"), + Arguments.of( + /* excludeRegexPattern= */ "nonexistent_xyz_pattern", + /* fromVersion= */ 1L, + /* toVersion= */ 4L, + /* expectedIds= */ new int[] {1, 2, 3, 4, 5, 6, 7, 8}, + /* isInitialSnapshot= */ false, + "Regex matching no files"), + Arguments.of( + /* excludeRegexPattern= */ "category=", + /* fromVersion= */ 1L, + /* toVersion= */ 4L, + /* expectedIds= */ new int[] {}, + /* isInitialSnapshot= */ false, + "Regex matching all files"), + Arguments.of( + /* excludeRegexPattern= */ "category=alpha", + /* fromVersion= */ 2L, + /* toVersion= */ 4L, + /* expectedIds= */ new int[] {3, 4, 7, 8}, + /* isInitialSnapshot= */ false, + "Exclude alpha versions 2 to 4"), + Arguments.of( + /* excludeRegexPattern= */ "category=beta", + /* fromVersion= */ 2L, + /* toVersion= */ 2L, + /* expectedIds= */ new int[] {}, + /* isInitialSnapshot= */ false, + "Exclude beta version 2 only"), + Arguments.of( + /* excludeRegexPattern= */ (String) null, + /* fromVersion= */ 0L, + /* toVersion= */ 4L, + /* expectedIds= */ new int[] {1, 2, 3, 4, 5, 6, 7, 8}, + /* isInitialSnapshot= */ true, + "Initial snapshot - no excludeRegex"), + Arguments.of( + /* excludeRegexPattern= */ "category=alpha", + /* fromVersion= */ 0L, + /* toVersion= */ 4L, + /* expectedIds= */ new int[] {3, 4, 7, 8}, + /* isInitialSnapshot= */ true, + "Initial snapshot - exclude alpha")); + } + /** * Helper method to convert InternalRow to Row for comparison. * @@ -2256,26 +2856,6 @@ public void testSchemaEvolution_onForwardNonAdditiveChanges_throwsError( + "_" + System.nanoTime(); - // TODO(#5318): Skip forward-fill drop column test until checkNonAdditiveSchemaChanges is - // implemented. - // - // `isReadCompatible` cannot distinguish between safe "backfill + add column" and breaking - // "forward-fill + drop column" scenarios. Both involve reading [A,B] data with [A,B,C] schema, - // which passes compatibility checks. However, the latter is a breaking change. - // - // Special handling is needed to detect "forward-fill + drop column" scenarios. DSv1 solves - // this via `checkNonAdditiveSchemaChanges`, which relies on - // DeltaColumnMapping.hasNoColumnMappingSchemaChanges. - // - // DSv1 has the same limitation when `checkNonAdditiveSchemaChanges` is removed. Since - // non-additive schema change detection is the next work item and provides the complete - // solution for all column mapping changes, implementing a partial fix now would create - // duplicate logic. DSv1 also behaved the same way before `checkNonAdditiveSchemaChanges` - // was implemented, so we skip this test as a short-term solution for now. - assumeFalse( - testDescription.contains("Drop"), - "Skipping drop column test - checkNonAdditiveSchemaChanges not yet implemented in v2 (TODO #5318)"); - createSchemaEvolutionTestTable(testTablePath, testTableName); // Try to read from version 0, which should include commits with METADATA actions @@ -2322,9 +2902,9 @@ public void testSchemaEvolution_onForwardNonAdditiveChanges_throwsError( }, String.format("DSv1 should throw on METADATA for scenario: %s", testDescription)); - DeltaIllegalStateException dsv2Exception = + DeltaUnsupportedOperationException dsv2Exception = assertThrows( - DeltaIllegalStateException.class, + DeltaUnsupportedOperationException.class, () -> { CloseableIterator kernelChanges = stream.getFileChanges( @@ -2349,7 +2929,23 @@ public void testSchemaEvolution_onForwardNonAdditiveChanges_throwsError( } }, String.format("DSv2 should throw on METADATA for scenario: %s", testDescription)); - // TODO(#5318): assert two exceptions are equal after schema tracking is supported in v2 + + // TODO(#5319): assertEqual after schema tracking log is supported + String expectedPrefix = "DELTA_STREAMING_INCOMPATIBLE_SCHEMA_CHANGE"; + assertTrue( + dsv1Exception.getErrorClass().startsWith(expectedPrefix), + String.format( + "v1 connector error class should start with %s, but got: %s", + expectedPrefix, dsv1Exception.getErrorClass())); + assertTrue( + dsv2Exception.getErrorClass().startsWith(expectedPrefix), + String.format( + "v2 connector error class should start with %s, but got: %s", + expectedPrefix, dsv2Exception.getErrorClass())); + assertEquals( + dsv1Exception.getMessageParameters(), + dsv2Exception.getMessageParameters(), + "v1 connector and v2 connector should throw the same error messages on forward-fill non-additive schema changes"); } finally { // recover spark config to original state sparkConf.forEach((key, value) -> spark.conf().unset(key)); @@ -2420,9 +3016,9 @@ public void testSchemaEvolution_onBackfillNonAdditiveChanges_throwsError( }, String.format("DSv1 should throw on METADATA for scenario: %s", testDescription)); - DeltaIllegalStateException dsv2Exception = + DeltaUnsupportedOperationException dsv2Exception = assertThrows( - DeltaIllegalStateException.class, + DeltaUnsupportedOperationException.class, () -> { CloseableIterator kernelChanges = stream.getFileChanges( @@ -2447,7 +3043,23 @@ public void testSchemaEvolution_onBackfillNonAdditiveChanges_throwsError( } }, String.format("DSv2 should throw on METADATA for scenario: %s", testDescription)); - // TODO(#5318): assert two exceptions are equal after schema tracking is supported in v2 + + // TODO(#5319): assertEqual after schema tracking log is supported + String expectedPrefix = "DELTA_STREAMING_INCOMPATIBLE_SCHEMA_CHANGE"; + assertTrue( + dsv1Exception.getErrorClass().startsWith(expectedPrefix), + String.format( + "v1 connector error class should start with %s, but got: %s", + expectedPrefix, dsv1Exception.getErrorClass())); + assertTrue( + dsv2Exception.getErrorClass().startsWith(expectedPrefix), + String.format( + "v2 connector error class should start with %s, but got: %s", + expectedPrefix, dsv2Exception.getErrorClass())); + assertEquals( + dsv1Exception.getMessageParameters(), + dsv2Exception.getMessageParameters(), + "v1 connector and v2 connector should throw the same error messages on backfill non-additive schema changes"); } finally { // recover spark config to original state sparkConf.forEach((key, value) -> spark.conf().unset(key)); @@ -2476,7 +3088,10 @@ public void testSchemaEvolution_onStreamStartOnce(@TempDir File tempDir) throws // Execute schema change before source initialization to ensure backfill change spark.sql(String.format("ALTER table %s RENAME COLUMN id TO userId", testTableName)); - spark.sql(String.format("INSERT INTO %s VALUES (3, 'Cathy', 5)", testTableName)); + spark.sql( + String.format( + "INSERT INTO %s VALUES (3, 'Cathy', 5, named_struct('col1', 18, 'col2', 'SF'))", + testTableName)); // Record the version prior to reverting schema change long incompatibleSchemaVersion = DeltaLog.forTable(spark, new Path(testTablePath)) @@ -2484,7 +3099,10 @@ public void testSchemaEvolution_onStreamStartOnce(@TempDir File tempDir) throws .version(); // Revert the schema change spark.sql(String.format("ALTER table %s RENAME COLUMN userId TO id", testTableName)); - spark.sql(String.format("INSERT INTO %s VALUES (4, 'David', 8)", testTableName)); + spark.sql( + String.format( + "INSERT INTO %s VALUES (4, 'David', 8, named_struct('col1', 47, 'col2', 'DC'))", + testTableName)); DeltaLog deltaLog = DeltaLog.forTable(spark, new Path(testTablePath)); String tableId = deltaLog.tableId(); @@ -2517,11 +3135,9 @@ public void testSchemaEvolution_onStreamStartOnce(@TempDir File tempDir) throws new PathBasedSnapshotManager(testTablePath, hadoopConf); SparkMicroBatchStream stream = createTestStreamWithDefaults(snapshotManager, hadoopConf, emptyDeltaOptions()); - // TODO(#5318): Change error type to - // DeltaErrors.blockStreamingReadsWithIncompatibleNonAdditiveSchemaChanges - DeltaIllegalStateException dsv2Exception = + DeltaUnsupportedOperationException dsv2Exception = assertThrows( - DeltaIllegalStateException.class, + DeltaUnsupportedOperationException.class, () -> stream.latestOffset(startOffset, readLimit), String.format( "DSv2 should throw error on stream start for scenario: %s", testDescription)); @@ -2530,6 +3146,23 @@ public void testSchemaEvolution_onStreamStartOnce(@TempDir File tempDir) throws .anyMatch( element -> element.toString().contains("checkReadIncompatibleSchemaChangeOnStreamStartOnce")); + + // TODO(#5319): assertEqual after schema tracking log is supported + String expectedPrefix = "DELTA_STREAMING_INCOMPATIBLE_SCHEMA_CHANGE"; + assertTrue( + dsv1Exception.getErrorClass().startsWith(expectedPrefix), + String.format( + "v1 connector error class should start with %s, but got: %s", + expectedPrefix, dsv1Exception.getErrorClass())); + assertTrue( + dsv2Exception.getErrorClass().startsWith(expectedPrefix), + String.format( + "v2 connector error class should start with %s, but got: %s", + expectedPrefix, dsv2Exception.getErrorClass())); + assertEquals( + dsv1Exception.getMessageParameters(), + dsv2Exception.getMessageParameters(), + "v1 connector and v2 connector should throw the same error messages on stream start schema changes"); } /** Provides test scenarios that generate additive schema changes actions. */ @@ -2629,18 +3262,31 @@ private static Stream nonAdditiveSchemaEvolutionScenarios() { /* sparkConf */ Map.of(), "Rename column"), - // Drop both nullable and non-nullable columns + // Drop nullable, non-nullable and struct columns Arguments.of( (ScenarioSetup) (tableName, tempDir) -> { - sql("ALTER TABLE %s DROP COLUMNS (id, value)", tableName); + sql("ALTER TABLE %s DROP COLUMNS (id, value, info)", tableName); }, /* sparkConf */ Map.of( DeltaSQLConf .DELTA_STREAMING_UNSAFE_READ_ON_INCOMPATIBLE_COLUMN_MAPPING_SCHEMA_CHANGES() .key(), "false"), - "Drop both nullable and non-nullable columns"), + "Drop nullable, non-nullable and struct columns"), + + // Drop column in nested struct + Arguments.of( + (ScenarioSetup) + (tableName, tempDir) -> { + sql("ALTER TABLE %s DROP COLUMNS info.col1", tableName); + }, + /* sparkConf */ Map.of( + DeltaSQLConf + .DELTA_STREAMING_UNSAFE_READ_ON_INCOMPATIBLE_COLUMN_MAPPING_SCHEMA_CHANGES() + .key(), + "false"), + "Drop column in nested struct"), // Widen INT column to BIGINT Arguments.of( @@ -3030,4 +3676,139 @@ public void testFindClosedByInterruptCause() { "readJsonFile", new java.io.FileNotFoundException("missing")))) .isEmpty(); } + + /** Regression test: closing the wrapped iterator must also close CommitActions. */ + @Test + public void testWrapIteratorWithCommitClose_closesCommitOnIteratorClose() throws Exception { + AtomicBoolean commitClosed = new AtomicBoolean(false); + AtomicBoolean innerClosed = new AtomicBoolean(false); + + CloseableIterator inner = + new CloseableIterator() { + private boolean consumed = false; + + @Override + public boolean hasNext() { + return !consumed; + } + + @Override + public IndexedFile next() { + consumed = true; + return new IndexedFile(/* version= */ 1L, /* index= */ 0L, /* addFile= */ null); + } + + @Override + public void close() { + innerClosed.set(true); + } + }; + + try (CloseableIterator wrapped = + SparkMicroBatchStream.wrapIteratorWithCommitClose( + inner, newTrackingCommitActions(commitClosed))) { + while (wrapped.hasNext()) { + wrapped.next(); + } + } + + assertTrue(innerClosed.get(), "Inner iterator should be closed"); + assertTrue(commitClosed.get(), "CommitActions should be closed"); + } + + /** Closing the wrapped iterator before full consumption must still close CommitActions. */ + @Test + public void testWrapIteratorWithCommitClose_closesCommitOnEarlyClose() throws Exception { + AtomicBoolean commitClosed = new AtomicBoolean(false); + + CloseableIterator inner = + new CloseableIterator() { + private int remaining = 10; + + @Override + public boolean hasNext() { + return remaining > 0; + } + + @Override + public IndexedFile next() { + remaining--; + return new IndexedFile(/* version= */ 1L, /* index= */ remaining, /* addFile= */ null); + } + + @Override + public void close() {} + }; + + try (CloseableIterator wrapped = + SparkMicroBatchStream.wrapIteratorWithCommitClose( + inner, newTrackingCommitActions(commitClosed))) { + assertTrue(wrapped.hasNext()); + wrapped.next(); + // Intentionally don't consume the rest + } + + assertTrue(commitClosed.get(), "CommitActions should be closed on early termination"); + } + + /** If inner.close() throws, CommitActions must still be closed. */ + @Test + public void testWrapIteratorWithCommitClose_closesCommitEvenWhenInnerCloseThrows() + throws Exception { + AtomicBoolean commitClosed = new AtomicBoolean(false); + + CloseableIterator inner = + new CloseableIterator() { + @Override + public boolean hasNext() { + return false; + } + + @Override + public IndexedFile next() { + throw new java.util.NoSuchElementException(); + } + + @Override + public void close() { + throw new RuntimeException("inner close failed"); + } + }; + + CloseableIterator wrapped = + SparkMicroBatchStream.wrapIteratorWithCommitClose( + inner, newTrackingCommitActions(commitClosed)); + try { + wrapped.close(); + } catch (RuntimeException e) { + // Expected — inner.close() threw + } + + assertTrue(commitClosed.get(), "CommitActions should be closed even when inner.close() throws"); + } + + /** Creates a CommitActions stub that sets {@code closedFlag} to true on close. */ + private static CommitActions newTrackingCommitActions(AtomicBoolean closedFlag) { + return new CommitActions() { + @Override + public long getVersion() { + return 1L; + } + + @Override + public long getTimestamp() { + return 0L; + } + + @Override + public CloseableIterator getActions() { + throw new UnsupportedOperationException("not needed for this test"); + } + + @Override + public void close() { + closedFlag.set(true); + } + }; + } } diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanBuilderTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanBuilderTest.java index f760f8d1775..f1884bfda44 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanBuilderTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanBuilderTest.java @@ -27,9 +27,11 @@ import io.delta.spark.internal.v2.snapshot.PathBasedSnapshotManager; import java.io.File; import java.lang.reflect.Field; +import java.math.BigDecimal; import java.util.Arrays; import java.util.HashSet; import java.util.Optional; +import java.util.Set; import org.apache.spark.sql.connector.read.Scan; import org.apache.spark.sql.connector.read.streaming.MicroBatchStream; import org.apache.spark.sql.sources.*; @@ -62,6 +64,13 @@ public void testBuild_returnsScanWithExpectedSchema(@TempDir File tempDir) { StructType partitionSchema = DataTypes.createStructType( new StructField[] {DataTypes.createStructField("dep_id", DataTypes.IntegerType, true)}); + StructType tableSchema = + DataTypes.createStructType( + new StructField[] { + DataTypes.createStructField("id", DataTypes.IntegerType, true), + DataTypes.createStructField("name", DataTypes.StringType, true), + DataTypes.createStructField("dep_id", DataTypes.IntegerType, true) + }); SparkScanBuilder builder = new SparkScanBuilder( tableName, @@ -69,6 +78,8 @@ public void testBuild_returnsScanWithExpectedSchema(@TempDir File tempDir) { snapshotManager, dataSchema, partitionSchema, + tableSchema, + Optional.empty(), CaseInsensitiveStringMap.empty()); StructType expectedSparkSchema = @@ -105,6 +116,13 @@ public void testToMicroBatchStream_returnsSparkMicroBatchStream(@TempDir File te StructType partitionSchema = DataTypes.createStructType( new StructField[] {DataTypes.createStructField("dep_id", DataTypes.IntegerType, true)}); + StructType tableSchema = + DataTypes.createStructType( + new StructField[] { + DataTypes.createStructField("id", DataTypes.IntegerType, true), + DataTypes.createStructField("name", DataTypes.StringType, true), + DataTypes.createStructField("dep_id", DataTypes.IntegerType, true) + }); SparkScanBuilder builder = new SparkScanBuilder( tableName, @@ -112,6 +130,8 @@ public void testToMicroBatchStream_returnsSparkMicroBatchStream(@TempDir File te snapshotManager, dataSchema, partitionSchema, + tableSchema, + Optional.empty(), CaseInsensitiveStringMap.empty()); Scan scan = builder.build(); @@ -150,15 +170,41 @@ public void testPushFilters_singleUnsupportedDataFilter(@TempDir File tempDir) t checkSupportsPushDownFilters( builder, - new Filter[] {new StringStartsWith("name", "test")}, // input filters - new Filter[] {new StringStartsWith("name", "test")}, // expected post-scan filters - new Filter[] {}, // expected pushed filters + new Filter[] {new StringEndsWith("name", "test")}, // input filters + new Filter[] { + new StringEndsWith("name", "test") + }, // expected post-scan filters (unsupported, stays for row-level eval) + new Filter[] {}, // expected pushed filters (nothing pushed) new Predicate[] {}, // expected pushed kernel predicates - new Filter[] {new StringStartsWith("name", "test")}, // expected data filters + new Filter[] {new StringEndsWith("name", "test")}, // expected data filters Optional.empty() // expected kernelScanBuilder.predicate ); } + @Test + public void testPushFilters_singleSupportedDataFilter_StringStartsWith(@TempDir File tempDir) + throws Exception { + SparkScanBuilder builder = createTestScanBuilder(tempDir); + + checkSupportsPushDownFilters( + builder, + new Filter[] {new StringStartsWith("name", "test")}, // input filters + new Filter[] { + new StringStartsWith("name", "test") + }, // expected post-scan filters (data filter still needs row-level eval) + new Filter[] {new StringStartsWith("name", "test")}, // expected pushed filters + new Predicate[] { + new Predicate("STARTS_WITH", new Column("name"), Literal.ofString("test")) + }, // expected pushed kernel predicates + new Filter[] {new StringStartsWith("name", "test")}, // expected data filters + Optional.of( + new Predicate( + "STARTS_WITH", + new Column("name"), + Literal.ofString("test"))) // expected kernelScanBuilder.predicate + ); + } + @Test public void testPushFilters_multiSupportedDataFilters(@TempDir File tempDir) throws Exception { SparkScanBuilder builder = createTestScanBuilder(tempDir); @@ -197,17 +243,17 @@ public void testPushFilters_mixedSupportedAndUnsupportedDataFilters(@TempDir Fil // input filters new Filter[] { new EqualTo("id", 100), // supported - new StringStartsWith("name", "test") // unsupported + new StringEndsWith("name", "test") // unsupported }, // expected post-scan filters - new Filter[] {new EqualTo("id", 100), new StringStartsWith("name", "test")}, - // expected pushed filters + new Filter[] {new EqualTo("id", 100), new StringEndsWith("name", "test")}, + // expected pushed filters (only the supported EqualTo is pushed) new Filter[] {new EqualTo("id", 100)}, // expected pushed kernel predicates new Predicate[] {new Predicate("=", new Column("id"), Literal.ofInt(100))}, // expected data filters - new Filter[] {new EqualTo("id", 100), new StringStartsWith("name", "test")}, - // expected kernelScanBuilder.predicate + new Filter[] {new EqualTo("id", 100), new StringEndsWith("name", "test")}, + // expected kernelScanBuilder.predicate (only EqualTo was pushed to kernel) Optional.of(new Predicate("=", new Column("id"), Literal.ofInt(100)))); } @@ -240,11 +286,17 @@ public void testPushFilters_singleUnsupportedPartitionFilter(@TempDir File tempD checkSupportsPushDownFilters( builder, new Filter[] {new StringStartsWith("dep_id", "1")}, // input filters - new Filter[] {new StringStartsWith("dep_id", "1")}, // expected - new Filter[] {}, // expected pushed filters - new Predicate[] {}, // expected pushed kernel predicates + new Filter[] {}, // expected post-scan filters (partition filter, fully pushed) + new Filter[] {new StringStartsWith("dep_id", "1")}, // expected pushed filters + new Predicate[] { + new Predicate("STARTS_WITH", new Column("dep_id"), Literal.ofString("1")) + }, // expected pushed kernel predicates new Filter[] {}, // expected data filters - Optional.empty() // expected kernelScanBuilder.predicate + Optional.of( + new Predicate( + "STARTS_WITH", + new Column("dep_id"), + Literal.ofString("1"))) // expected kernelScanBuilder.predicate ); } @@ -290,15 +342,22 @@ public void testPushFilters_mixedSupportedAndUnsupportedPartitionFilters(@TempDi new StringStartsWith("dep_id", "1") // unsupported }, // expected post-scan filters - new Filter[] {new StringStartsWith("dep_id", "1")}, + new Filter[] {}, // expected pushed filters - new Filter[] {new EqualTo("dep_id", 1)}, + new Filter[] {new EqualTo("dep_id", 1), new StringStartsWith("dep_id", "1")}, // expected pushed kernel predicates - new Predicate[] {new Predicate("=", new Column("dep_id"), Literal.ofInt(1))}, + new Predicate[] { + new Predicate("=", new Column("dep_id"), Literal.ofInt(1)), + new Predicate("STARTS_WITH", new Column("dep_id"), Literal.ofString("1")) + }, // expected data filters new Filter[] {}, // expected kernelScanBuilder.predicate - Optional.of(new Predicate("=", new Column("dep_id"), Literal.ofInt(1)))); + Optional.of( + new Predicate( + "AND", + new Predicate("=", new Column("dep_id"), Literal.ofInt(1)), + new Predicate("STARTS_WITH", new Column("dep_id"), Literal.ofString("1"))))); } @Test @@ -323,25 +382,29 @@ public void testPushFilters_mixedFilters(@TempDir File tempDir) throws Exception // expected pushed filters new Filter[] { new EqualTo("id", 100), // data filter, supported + new StringStartsWith("name", "foo"), // data filter, supported new GreaterThan("dep_id", 1) // partition filter, supported }, // expected pushed kernel predicates new Predicate[] { new Predicate("=", new Column("id"), Literal.ofInt(100)), + new Predicate("STARTS_WITH", new Column("name"), Literal.ofString("foo")), new Predicate(">", new Column("dep_id"), Literal.ofInt(1)) }, // expected data filters new Filter[] { new EqualTo("id", 100), // data filter, supported - new StringStartsWith("name", "foo") // data filter, unsupported + new StringStartsWith("name", "foo") // data filter, supported }, // expected kernelScanBuilder.predicate Optional.of( new Predicate( "AND", - Arrays.asList( + new Predicate( + "AND", new Predicate("=", new Column("id"), Literal.ofInt(100)), - new Predicate(">", new Column("dep_id"), Literal.ofInt(1)))))); + new Predicate("STARTS_WITH", new Column("name"), Literal.ofString("foo"))), + new Predicate(">", new Column("dep_id"), Literal.ofInt(1))))); } @Test @@ -380,18 +443,20 @@ public void testPushFilters_ORSupportedAndUnsupportedDataFilters(@TempDir File t throws Exception { SparkScanBuilder builder = createTestScanBuilder(tempDir); + // OR(supported, unsupported) cannot be partially pushed: if one branch is unsupported, + // the whole OR must remain for post-scan evaluation and nothing is pushed to the kernel. checkSupportsPushDownFilters( builder, // input filters - new Filter[] {new Or(new EqualTo("id", 100), new StringStartsWith("name", "foo"))}, - // expected post-scan filters - new Filter[] {new Or(new EqualTo("id", 100), new StringStartsWith("name", "foo"))}, - // expected pushed filters + new Filter[] {new Or(new EqualTo("id", 100), new StringEndsWith("name", "foo"))}, + // expected post-scan filters (whole OR stays, since one branch is unsupported) + new Filter[] {new Or(new EqualTo("id", 100), new StringEndsWith("name", "foo"))}, + // expected pushed filters (nothing pushed) new Filter[] {}, // expected pushed kernel predicates new Predicate[] {}, // expected data filters - new Filter[] {new Or(new EqualTo("id", 100), new StringStartsWith("name", "foo"))}, + new Filter[] {new Or(new EqualTo("id", 100), new StringEndsWith("name", "foo"))}, // expected kernelScanBuilder.predicate Optional.empty()); } @@ -431,21 +496,20 @@ public void testPushFilters_ORSupportedDataAndPartitionFilters(@TempDir File tem } /* - * (supportedPartitionFilterA AND unsupportedPartitionFilterB) OR supportedPartitionFilterC - * will be converted to A OR C, because of the partial pushdown of AND + * (partitionFilterA AND partitionFilterB) OR partitionFilterC + * where A = EqualTo("dep_id", 1), B = StringStartsWith("dep_id", "1"), C = GreaterThan("dep_id", 2) + * All three are fully supported partition filters, so the whole expression is pushed down. * - * Expected post-scan filters: (A AND B) OR C - * Expected pushed filters: none (TODO: should be sparkFilterA OR sparkFilterC) - * Expected pushed kernel predicates: predicateA OR predicateC + * Expected post-scan filters: none (all partition filters, fully pushed) + * Expected pushed filters: (A AND B) OR C + * Expected pushed kernel predicates: (predicateA AND predicateB) OR predicateC * Expected data filters: none - * Expected kernelScanBuilder.predicate: predicateA OR predicateC + * Expected kernelScanBuilder.predicate: (predicateA AND predicateB) OR predicateC */ @Test public void testPushFilters_mixedORandAND(@TempDir File tempDir) throws Exception { SparkScanBuilder builder = createTestScanBuilder(tempDir); - // (supportedPartitionFilterA AND unsupportedPartitionFilterB) OR supportedPartitionFilterC - checkSupportsPushDownFilters( builder, // input filters @@ -455,19 +519,24 @@ public void testPushFilters_mixedORandAND(@TempDir File tempDir) throws Exceptio new GreaterThan("dep_id", 2)) }, // expected post-scan filters + new Filter[] {}, + // expected pushed filters new Filter[] { new Or( new And(new EqualTo("dep_id", 1), new StringStartsWith("dep_id", "1")), new GreaterThan("dep_id", 2)) }, - // expected pushed filters - new Filter[] {}, // expected pushed kernel predicates new Predicate[] { new Predicate( "OR", Arrays.asList( - new Predicate("=", new Column("dep_id"), Literal.ofInt(1)), + new Predicate( + "AND", + Arrays.asList( + new Predicate("=", new Column("dep_id"), Literal.ofInt(1)), + new Predicate( + "STARTS_WITH", new Column("dep_id"), Literal.ofString("1")))), new Predicate(">", new Column("dep_id"), Literal.ofInt(2)))) }, // expected data filters @@ -477,7 +546,12 @@ public void testPushFilters_mixedORandAND(@TempDir File tempDir) throws Exceptio new Predicate( "OR", Arrays.asList( - new Predicate("=", new Column("dep_id"), Literal.ofInt(1)), + new Predicate( + "AND", + Arrays.asList( + new Predicate("=", new Column("dep_id"), Literal.ofInt(1)), + new Predicate( + "STARTS_WITH", new Column("dep_id"), Literal.ofString("1")))), new Predicate(">", new Column("dep_id"), Literal.ofInt(2)))))); } @@ -612,20 +686,23 @@ public void testPushFilters_NOTSupportedDataORUnsupportedDataFilters(@TempDir Fi throws Exception { SparkScanBuilder builder = createTestScanBuilder(tempDir); + // NOT(OR(supported, unsupported)): the OR branch contains an unsupported filter + // (StringEndsWith), + // so the whole NOT(OR(...)) cannot be pushed to the kernel. checkSupportsPushDownFilters( builder, // input filters new Filter[] { - new Not(new Or(new EqualTo("id", 100), new StringStartsWith("name", "foo"))), + new Not(new Or(new EqualTo("id", 100), new StringEndsWith("name", "foo"))), }, - // expected post-scan filters - new Filter[] {new Not(new Or(new EqualTo("id", 100), new StringStartsWith("name", "foo")))}, - // expected pushed filters + // expected post-scan filters (whole NOT(OR) stays, unsupported branch blocks pushdown) + new Filter[] {new Not(new Or(new EqualTo("id", 100), new StringEndsWith("name", "foo")))}, + // expected pushed filters (nothing pushed) new Filter[] {}, // expected pushed kernel predicates new Predicate[] {}, // expected data filters - new Filter[] {new Not(new Or(new EqualTo("id", 100), new StringStartsWith("name", "foo")))}, + new Filter[] {new Not(new Or(new EqualTo("id", 100), new StringEndsWith("name", "foo")))}, // expected kernelScanBuilder.predicate Optional.empty()); } @@ -664,15 +741,6 @@ private void checkSupportsPushDownFilters( } private SparkScanBuilder createTestScanBuilder(File tempDir) { - String path = tempDir.getAbsolutePath(); - String tableName = String.format("predicate_test_%d", System.currentTimeMillis()); - spark.sql( - String.format( - "CREATE OR REPLACE TABLE %s (id INT, name STRING, dep_id INT) USING delta PARTITIONED BY (dep_id) LOCATION '%s'", - tableName, path)); - PathBasedSnapshotManager snapshotManager = - new PathBasedSnapshotManager(path, spark.sessionState().newHadoopConf()); - Snapshot snapshot = snapshotManager.loadLatestSnapshot(); StructType dataSchema = DataTypes.createStructType( new StructField[] { @@ -682,12 +750,126 @@ private SparkScanBuilder createTestScanBuilder(File tempDir) { StructType partitionSchema = DataTypes.createStructType( new StructField[] {DataTypes.createStructField("dep_id", DataTypes.IntegerType, true)}); + StructType tableSchema = + DataTypes.createStructType( + new StructField[] { + DataTypes.createStructField("id", DataTypes.IntegerType, true), + DataTypes.createStructField("name", DataTypes.StringType, true), + DataTypes.createStructField("dep_id", DataTypes.IntegerType, true) + }); + return createScanBuilder(tempDir, dataSchema, partitionSchema, tableSchema); + } + + /** + * Integration test: decimal widening end-to-end through pushFilters() → classifyFilter() → + * convertComparisonLiteral(). Verifies that a decimal literal Decimal(5,2) is widened to match + * column type Decimal(7,2) when pushed through the full filter pushdown path. + */ + @Test + public void testPushFilters_decimalWideningEndToEnd(@TempDir File tempDir) throws Exception { + SparkScanBuilder builder = createDecimalTestScanBuilder(tempDir); + + // price = 100.00 where literal is Decimal(5,2) and column is Decimal(7,2) + Filter[] sparkFilter = new Filter[] {new EqualTo("price", new BigDecimal("100.00"))}; + Predicate kernelPredicate = + new Predicate("=", new Column("price"), Literal.ofDecimal(new BigDecimal("100.00"), 7, 2)); + + checkSupportsPushDownFilters( + builder, + sparkFilter, // input filters + sparkFilter, // expected post-scan filters (data filter, stays for row-level eval) + sparkFilter, // expected pushed filters + new Predicate[] {kernelPredicate}, // expected pushed kernel predicates (widened) + sparkFilter, // expected data filters + Optional.of(kernelPredicate)); // expected kernelScanBuilder.predicate + } + + /** + * Integration test: decimal literal with scale exceeding column scale is rejected during + * pushFilters(). The filter AND(price > 99.999, price < 200.00) should partially push only the + * right side because 99.999 has scale=3 exceeding column's Decimal(7,2) scale=2. + */ + @Test + public void testPushFilters_decimalRejectionPartialPushDown(@TempDir File tempDir) + throws Exception { + SparkScanBuilder builder = createDecimalTestScanBuilder(tempDir); + + // AND(price > 99.999, price < 200.00): left side has scale=3 exceeding column's scale=2 + Filter[] sparkFilter = + new Filter[] { + new And( + new GreaterThan("price", new BigDecimal("99.999")), + new LessThan("price", new BigDecimal("200.00"))) + }; + // Only the right side (price < 200.00) is pushed as a kernel predicate + Predicate kernelPredicate = + new Predicate("<", new Column("price"), Literal.ofDecimal(new BigDecimal("200.00"), 7, 2)); + + checkSupportsPushDownFilters( + builder, + sparkFilter, // input filters + sparkFilter, // expected post-scan filters (partial conversion, stays for row-level eval) + new Filter[] {}, // expected pushed filters (partial: Spark filter not added) + new Predicate[] {kernelPredicate}, // expected pushed kernel predicates + sparkFilter, // expected data filters + Optional.of(kernelPredicate)); // expected kernelScanBuilder.predicate + } + + private SparkScanBuilder createDecimalTestScanBuilder(File tempDir) { + StructType dataSchema = + DataTypes.createStructType( + new StructField[] { + DataTypes.createStructField("price", DataTypes.createDecimalType(7, 2), true), + DataTypes.createStructField("quantity", DataTypes.IntegerType, true) + }); + StructType partitionSchema = + DataTypes.createStructType( + new StructField[] {DataTypes.createStructField("dep_id", DataTypes.IntegerType, true)}); + StructType tableSchema = + DataTypes.createStructType( + new StructField[] { + DataTypes.createStructField("price", DataTypes.createDecimalType(7, 2), true), + DataTypes.createStructField("quantity", DataTypes.IntegerType, true), + DataTypes.createStructField("dep_id", DataTypes.IntegerType, true) + }); + return createScanBuilder(tempDir, dataSchema, partitionSchema, tableSchema); + } + + /** + * Shared helper for creating a SparkScanBuilder with the given schemas. Both + * createTestScanBuilder and createDecimalTestScanBuilder delegate to this method to avoid + * duplicating snapshot loading and builder instantiation logic. + */ + private SparkScanBuilder createScanBuilder( + File tempDir, StructType dataSchema, StructType partitionSchema, StructType tableSchema) { + String path = tempDir.getAbsolutePath(); + String tableName = String.format("test_%d", System.currentTimeMillis()); + // Build CREATE TABLE SQL from the tableSchema and partitionSchema + StringBuilder columns = new StringBuilder(); + Set partitionCols = new HashSet<>(); + for (StructField f : partitionSchema.fields()) { + partitionCols.add(f.name()); + } + for (StructField f : tableSchema.fields()) { + if (columns.length() > 0) columns.append(", "); + columns.append(f.name()).append(" ").append(f.dataType().sql()); + } + String partitionColNames = String.join(", ", partitionCols); + spark.sql( + String.format( + "CREATE OR REPLACE TABLE %s (%s) USING delta PARTITIONED BY (%s) LOCATION '%s'", + tableName, columns, partitionColNames, path)); + PathBasedSnapshotManager snapshotManager = + new PathBasedSnapshotManager(path, spark.sessionState().newHadoopConf()); + Snapshot snapshot = snapshotManager.loadLatestSnapshot(); return new SparkScanBuilder( tableName, snapshot, snapshotManager, dataSchema, partitionSchema, + tableSchema, + Optional.empty(), CaseInsensitiveStringMap.empty()); } diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanTest.java index e158a4a6539..2c452aa5461 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/SparkScanTest.java @@ -9,19 +9,36 @@ import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.hadoop.conf.Configuration; +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.catalyst.TableIdentifier; +import org.apache.spark.sql.catalyst.catalog.CatalogColumnStat; +import org.apache.spark.sql.catalyst.catalog.CatalogStatistics; +import org.apache.spark.sql.catalyst.catalog.CatalogTable; import org.apache.spark.sql.connector.catalog.Identifier; import org.apache.spark.sql.connector.expressions.Expression; import org.apache.spark.sql.connector.expressions.FieldReference; import org.apache.spark.sql.connector.expressions.LiteralValue; +import org.apache.spark.sql.connector.expressions.NamedReference; import org.apache.spark.sql.connector.expressions.filter.Predicate; +import org.apache.spark.sql.connector.read.Batch; +import org.apache.spark.sql.connector.read.HasPartitionKey; +import org.apache.spark.sql.connector.read.InputPartition; import org.apache.spark.sql.connector.read.Scan; import org.apache.spark.sql.connector.read.ScanBuilder; +import org.apache.spark.sql.connector.read.Statistics; +import org.apache.spark.sql.connector.read.colstats.ColumnStatistics; +import org.apache.spark.sql.connector.read.partitioning.KeyGroupedPartitioning; +import org.apache.spark.sql.connector.read.partitioning.Partitioning; +import org.apache.spark.sql.connector.read.partitioning.UnknownPartitioning; import org.apache.spark.sql.delta.DeltaOptions; +import org.apache.spark.sql.execution.datasources.FilePartition; import org.apache.spark.sql.execution.datasources.PartitionedFile; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructType; @@ -95,6 +112,131 @@ public static void setupPartitionedTable(@TempDir File tempDir) { protected static final List allCities = Arrays.asList("city=hz", "city=sh", "city=bj", "city=sz"); + // =============================================================================================== + // Tests for columnarSupportMode + // =============================================================================================== + + @Test + public void testColumnarSupportModeReturnsSupported() { + // Table schema uses simple types (INT, STRING) which are batch-read-compatible + SparkScanBuilder builder = (SparkScanBuilder) table.newScanBuilder(options); + SparkScan scan = (SparkScan) builder.build(); + + assertEquals( + Scan.ColumnarSupportMode.SUPPORTED, + scan.columnarSupportMode(), + "columnarSupportMode should return SUPPORTED for batch-compatible schema"); + } + + @Test + public void testColumnarSupportModeDoesNotTriggerPlanning() throws Exception { + // Calling columnarSupportMode() must NOT trigger file planning (the whole point of the + // override is to avoid the early planInputPartitions() call that PARTITION_DEFINED causes). + SparkScanBuilder builder = (SparkScanBuilder) table.newScanBuilder(options); + SparkScan scan = (SparkScan) builder.build(); + + // Call columnarSupportMode before any planning + scan.columnarSupportMode(); + + // Verify the scan has not been planned yet + Field plannedField = SparkScan.class.getDeclaredField("planned"); + plannedField.setAccessible(true); + assertFalse( + (boolean) plannedField.get(scan), "columnarSupportMode() should not trigger file planning"); + } + + @Test + public void testColumnarSupportModeWithUnsupportedSchema(@TempDir File tempDir) throws Exception { + // Create a table with a MAP column and disable nested column vectorized reading + // to ensure the schema is not batch-read-compatible + String path = tempDir.getAbsolutePath(); + String mapTableName = "columnar_map_table"; + + withTable( + new String[] {mapTableName}, + () -> { + spark.sql( + String.format( + "CREATE TABLE %s (id INT, tags MAP) USING delta LOCATION '%s'", + mapTableName, path)); + spark.sql(String.format("INSERT INTO %s VALUES (1, map('k', 'v'))", mapTableName)); + + withSQLConf( + "spark.sql.parquet.enableNestedColumnVectorizedReader", + "false", + () -> { + SparkTable mapTable = + new SparkTable( + Identifier.of(new String[] {"spark_catalog", "default"}, mapTableName), + path, + options); + SparkScanBuilder builder = (SparkScanBuilder) mapTable.newScanBuilder(options); + SparkScan scan = (SparkScan) builder.build(); + + assertEquals( + Scan.ColumnarSupportMode.UNSUPPORTED, + scan.columnarSupportMode(), + "columnarSupportMode should return UNSUPPORTED for schema with MAP type" + + " when nested column vectorized reader is disabled"); + }); + }); + } + + @Test + public void testColumnarSupportModeWithVectorizedReaderDisabled() throws Exception { + // When spark.sql.parquet.enableVectorizedReader is false, columnarSupportMode should + // return UNSUPPORTED even for batch-compatible schemas. + withSQLConf( + "spark.sql.parquet.enableVectorizedReader", + "false", + () -> { + SparkScanBuilder builder = (SparkScanBuilder) table.newScanBuilder(options); + SparkScan scan = (SparkScan) builder.build(); + + assertEquals( + Scan.ColumnarSupportMode.UNSUPPORTED, + scan.columnarSupportMode(), + "columnarSupportMode should return UNSUPPORTED when vectorized reader is disabled"); + }); + } + + @Test + public void testColumnarSupportModeWithDeletionVectors(@TempDir File tempDir) throws Exception { + // For a DV-enabled table with a batch-compatible schema, columnarSupportMode should still + // return SUPPORTED because the DV internal column (__delta_internal_is_row_deleted, ByteType) + // is also batch-compatible. This verifies consistency with PartitionUtils reader factory. + String dvPath = tempDir.getAbsolutePath(); + String dvTableName = "columnar_dv_table"; + + withTable( + new String[] {dvTableName}, + () -> { + spark.sql( + String.format( + "CREATE TABLE %s (id INT, value STRING) USING delta " + + "TBLPROPERTIES ('delta.enableDeletionVectors' = 'true') " + + "LOCATION '%s'", + dvTableName, dvPath)); + spark.sql(String.format("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", dvTableName)); + + SparkTable dvTable = + new SparkTable( + Identifier.of(new String[] {"spark_catalog", "default"}, dvTableName), + dvPath, + options); + SparkScanBuilder builder = (SparkScanBuilder) dvTable.newScanBuilder(options); + SparkScan scan = (SparkScan) builder.build(); + + // DV-enabled table with simple types should still return SUPPORTED because the + // DV column (ByteType) is batch-compatible + assertEquals( + Scan.ColumnarSupportMode.SUPPORTED, + scan.columnarSupportMode(), + "columnarSupportMode should return SUPPORTED for DV-enabled table with" + + " batch-compatible schema"); + }); + } + // =============================================================================================== // Tests for getDataSchema, getPartitionSchema, getReadDataSchema, getOptions, getConfiguration // =============================================================================================== @@ -445,7 +587,8 @@ public void testValidateStreamingOptions_UnsupportedOptions() { // Note: DeltaOptions uses CaseInsensitiveMap which lowercases keys during iteration assertEquals( "The following streaming options are not supported: [readchangefeed]. " - + "Supported options are: [startingVersion, startingTimestamp, maxFilesPerTrigger, maxBytesPerTrigger].", + + "Supported options are: [startingVersion, startingTimestamp, maxFilesPerTrigger, " + + "maxBytesPerTrigger, ignoreChanges, ignoreDeletes, skipChangeCommits, excludeRegex].", exception.getMessage()); } @@ -810,4 +953,647 @@ public void testNotEqualsWithAndWithoutRuntimeFilter() { assertNotEquals(scan1, scan2); assertNotEquals(scan1.hashCode(), scan2.hashCode()); } + + // ================================================================================================ + // Tests for output partitioning (SupportsReportPartitioning) + // ================================================================================================ + + @Test + public void testOutputPartitioningForPartitionedTable() { + // Partitioned table should return KeyGroupedPartitioning + SparkScanBuilder builder = (SparkScanBuilder) table.newScanBuilder(options); + SparkScan scan = (SparkScan) builder.build(); + + Partitioning partitioning = scan.outputPartitioning(); + + assertTrue( + partitioning instanceof KeyGroupedPartitioning, + "Partitioned table should return KeyGroupedPartitioning"); + + KeyGroupedPartitioning kgp = (KeyGroupedPartitioning) partitioning; + + // The partitioned table has 3 partition columns: date, city, part + Expression[] keys = kgp.keys(); + assertEquals(3, keys.length, "Should have 3 partition key expressions"); + + // Verify partition key names match partition schema + Set keyNames = new HashSet<>(); + for (Expression key : keys) { + assertTrue(key instanceof FieldReference, "Key should be a FieldReference"); + keyNames.add(((FieldReference) key).fieldNames()[0]); + } + assertTrue(keyNames.containsAll(Arrays.asList("date", "city", "part"))); + + // numPartitions returns partitionedFiles.size() (file count, not unique partition count). + // In this test data, each partition has one file, so file count equals partition count. + assertEquals(5, kgp.numPartitions(), "Should have 5 files (one per partition)"); + } + + /** Creates a non-partitioned table with sample data and returns a SparkScan for it. */ + private SparkScan createNonPartitionedScan(File tempDir, String tableName) { + spark.sql( + String.format( + "CREATE TABLE `%s` (id INT, name STRING) USING delta LOCATION '%s'", + tableName, tempDir.getAbsolutePath())); + spark.sql(String.format("INSERT INTO %s VALUES (1, 'Alice'), (2, 'Bob')", tableName)); + + SparkTable nonPartTable = + new SparkTable( + Identifier.of(new String[] {"spark_catalog", "default"}, tableName), + tempDir.getAbsolutePath(), + options); + + SparkScanBuilder builder = (SparkScanBuilder) nonPartTable.newScanBuilder(options); + return (SparkScan) builder.build(); + } + + @Test + public void testOutputPartitioningForNonPartitionedTable(@TempDir File tempDir) { + // Non-partitioned table should return UnknownPartitioning + SparkScan scan = createNonPartitionedScan(tempDir, "deltatbl_nonpartitioned"); + + Partitioning partitioning = scan.outputPartitioning(); + + assertTrue( + partitioning instanceof UnknownPartitioning, + "Non-partitioned table should return UnknownPartitioning"); + } + + @Test + public void testOutputPartitioningAfterRuntimeFilter() { + // Output partitioning should reflect filtered partition count + SparkScanBuilder builder = (SparkScanBuilder) table.newScanBuilder(options); + SparkScan scan = (SparkScan) builder.build(); + + // Apply filter to only keep city=hz (2 rows: part=1/date=20180520 and part=1/date=20180718) + scan.filter(new Predicate[] {cityPredicate}); + + Partitioning partitioning = scan.outputPartitioning(); + assertTrue(partitioning instanceof KeyGroupedPartitioning); + + KeyGroupedPartitioning kgp = (KeyGroupedPartitioning) partitioning; + // numPartitions returns partitionedFiles.size() (file count); here each partition has one file. + assertEquals( + 2, + kgp.numPartitions(), + "After filtering to city=hz, should have 2 files (one per partition)"); + } + + // ================================================================================================ + // Tests for DeltaInputPartition in planInputPartitions + // ================================================================================================ + + @Test + public void testPlanInputPartitionsReturnsHasPartitionKeyForPartitionedTable() { + SparkScanBuilder builder = (SparkScanBuilder) table.newScanBuilder(options); + SparkScan scan = (SparkScan) builder.build(); + Batch batch = scan.toBatch(); + + InputPartition[] partitions = batch.planInputPartitions(); + + assertTrue(partitions.length > 0, "Should have at least one partition"); + for (InputPartition partition : partitions) { + assertTrue( + partition instanceof DeltaInputPartition, + "Partitioned table should return DeltaInputPartition instances"); + assertTrue( + partition instanceof HasPartitionKey, + "DeltaInputPartition should implement HasPartitionKey"); + + DeltaInputPartition deltaPartition = (DeltaInputPartition) partition; + assertNotNull(deltaPartition.partitionKey(), "Partition key should not be null"); + assertNotNull(deltaPartition.getFilePartition(), "FilePartition should not be null"); + } + } + + @Test + public void testPlanInputPartitionsGroupsFilesByPartition(@TempDir File tempDir) + throws Exception { + // Create a table with multiple files per partition to actually exercise the grouping logic + // in planPartitionedInputPartitions (the default test table has 1 file per partition, + // which would pass even without grouping). + String multiFileTableName = "deltatbl_multifile_partitioned"; + spark.sql( + String.format( + "CREATE TABLE `%s` (id INT, data STRING, part INT) USING delta " + + "LOCATION '%s' PARTITIONED BY (part)", + multiFileTableName, tempDir.getAbsolutePath())); + // Insert in separate statements to create multiple files per partition + spark.sql( + String.format("INSERT INTO `%s` VALUES (1, 'a', 1), (2, 'b', 2)", multiFileTableName)); + spark.sql( + String.format("INSERT INTO `%s` VALUES (3, 'c', 1), (4, 'd', 2)", multiFileTableName)); + spark.sql(String.format("INSERT INTO `%s` VALUES (5, 'e', 1)", multiFileTableName)); + // Now part=1 has 3 files, part=2 has 2 files + + SparkTable multiFileTable = + new SparkTable( + Identifier.of(new String[] {"spark_catalog", "default"}, multiFileTableName), + tempDir.getAbsolutePath(), + options); + + // Force maxPartitionBytes=1 so each file gets its own FilePartition, making the + // totalPartitions > 2 assertion deterministic regardless of default parallelism. + withSQLConf( + "spark.sql.files.maxPartitionBytes", + "1", + () -> { + SparkScanBuilder builder = (SparkScanBuilder) multiFileTable.newScanBuilder(options); + SparkScan scan = (SparkScan) builder.build(); + Batch batch = scan.toBatch(); + + InputPartition[] partitions = batch.planInputPartitions(); + + // Verify all partitions are DeltaInputPartition with partition keys + Map> partitionsByKey = new HashMap<>(); + for (InputPartition p : partitions) { + assertTrue(p instanceof DeltaInputPartition); + DeltaInputPartition dp = (DeltaInputPartition) p; + partitionsByKey.computeIfAbsent(dp.partitionKey(), k -> new ArrayList<>()).add(dp); + } + + // Should have exactly 2 unique partition keys (part=1 and part=2) + assertEquals(2, partitionsByKey.size(), "Should have 2 unique partition keys"); + + // Verify that the grouping actually produced multiple DeltaInputPartitions for a + // single partition key (since multiple files exist per partition and each gets its + // own FilePartition when maxPartitionBytes=1) + int totalPartitions = partitions.length; + assertTrue( + totalPartitions > 2, + "With 5 files across 2 partitions, should have more than 2 input partitions, " + + "got " + + totalPartitions); + + // Verify all DeltaInputPartitions with the same key share the same partition key + for (Map.Entry> entry : + partitionsByKey.entrySet()) { + List group = entry.getValue(); + InternalRow expectedKey = group.get(0).partitionKey(); + for (DeltaInputPartition dp : group) { + assertEquals( + expectedKey, + dp.partitionKey(), + "All partitions in the same group should have equal partition keys"); + } + } + }); + } + + @Test + public void testPlanInputPartitionsReturnsFilePartitionForNonPartitionedTable( + @TempDir File tempDir) { + SparkScan scan = createNonPartitionedScan(tempDir, "deltatbl_nonpartitioned_batch"); + Batch batch = scan.toBatch(); + + InputPartition[] partitions = batch.planInputPartitions(); + + assertTrue(partitions.length > 0, "Should have at least one partition"); + for (InputPartition partition : partitions) { + assertTrue( + partition instanceof FilePartition, + "Non-partitioned table should return FilePartition instances, not DeltaInputPartition"); + assertFalse( + partition instanceof DeltaInputPartition, + "Non-partitioned table should NOT return DeltaInputPartition"); + } + } + + @Test + public void testOutputPartitioningForEmptyPartitionedTable(@TempDir File tempDir) { + // Empty partitioned table should return KeyGroupedPartitioning with 0 partitions + String emptyTableName = "deltatbl_empty_partitioned"; + spark.sql( + String.format( + "CREATE TABLE `%s` (id INT, name STRING, part INT) USING delta " + + "LOCATION '%s' PARTITIONED BY (part)", + emptyTableName, tempDir.getAbsolutePath())); + + SparkTable emptyTable = + new SparkTable( + Identifier.of(new String[] {"spark_catalog", "default"}, emptyTableName), + tempDir.getAbsolutePath(), + options); + + SparkScanBuilder builder = (SparkScanBuilder) emptyTable.newScanBuilder(options); + SparkScan scan = (SparkScan) builder.build(); + + Partitioning partitioning = scan.outputPartitioning(); + assertTrue( + partitioning instanceof KeyGroupedPartitioning, + "Empty partitioned table should still return KeyGroupedPartitioning"); + + KeyGroupedPartitioning kgp = (KeyGroupedPartitioning) partitioning; + assertEquals(0, kgp.numPartitions(), "Empty table should have 0 partitions"); + + Batch batch = scan.toBatch(); + InputPartition[] partitions = batch.planInputPartitions(); + assertEquals(0, partitions.length, "Empty table should return 0 input partitions"); + } + + // ================================================================================================ + // Tests for catalog statistics propagation + // ================================================================================================ + + /** + * Helper to inject CatalogStatistics into a catalog table via alterTableStats. This is needed + * because ANALYZE TABLE is not supported for V2 tables in the test environment. + */ + private CatalogTable injectCatalogStats(String tblName, CatalogStatistics stats) + throws Exception { + TableIdentifier tableId = new TableIdentifier(tblName); + spark.sessionState().catalog().alterTableStats(tableId, scala.Option.apply(stats)); + return spark.sessionState().catalog().getTableMetadata(tableId); + } + + @Test + public void testEstimateStatisticsWithCatalogStats_cboEnabled(@TempDir File tempDir) + throws Exception { + String path = tempDir.getAbsolutePath(); + String tblName = "stats_cbo_enabled"; + spark.sql( + String.format( + "CREATE TABLE %s (id INT, name STRING, value DOUBLE) USING delta LOCATION '%s'", + tblName, path)); + spark.sql(String.format("INSERT INTO %s VALUES (1, 'a', 1.0), (2, 'b', 2.0)", tblName)); + + // Inject catalog stats with column stats for "id" + CatalogColumnStat idColStat = + new CatalogColumnStat( + scala.Option.apply(scala.math.BigInt.apply(2L)), // distinctCount + scala.Option.apply("1"), // min + scala.Option.apply("2"), // max + scala.Option.apply(scala.math.BigInt.apply(0L)), // nullCount + scala.Option.apply((Object) 4L), // avgLen + scala.Option.apply((Object) 4L), // maxLen + scala.Option.empty(), // histogram + CatalogColumnStat.VERSION()); + + CatalogStatistics catalogStats = + new CatalogStatistics( + scala.math.BigInt.apply(1024L), + scala.Option.apply(scala.math.BigInt.apply(2L)), + buildColStatsMap(new String[] {"id"}, new CatalogColumnStat[] {idColStat})); + + CatalogTable catalogTable = injectCatalogStats(tblName, catalogStats); + + withSQLConf( + "spark.sql.cbo.enabled", + "true", + () -> { + Identifier id = Identifier.of(new String[] {"default"}, tblName); + SparkTable sparkTable = new SparkTable(id, catalogTable, Collections.emptyMap()); + + SparkScanBuilder builder = + (SparkScanBuilder) + sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); + SparkScan scan = (SparkScan) builder.build(); + Statistics stats = scan.estimateStatistics(); + + // Should have numRows from catalog stats + assertTrue(stats.numRows().isPresent(), "numRows should be present with CBO enabled"); + assertEquals(2L, stats.numRows().getAsLong(), "numRows should be 2"); + + // sizeInBytes should still come from planned files (more accurate) + assertTrue(stats.sizeInBytes().isPresent(), "sizeInBytes should be present"); + assertTrue(stats.sizeInBytes().getAsLong() > 0, "sizeInBytes should be positive"); + + // Should have column stats + Map colStats = stats.columnStats(); + assertNotNull(colStats, "columnStats should not be null"); + assertFalse(colStats.isEmpty(), "columnStats should not be empty"); + + // Check that column stats contain expected columns + ColumnStatistics idStats = colStats.get(FieldReference.apply("id")); + assertNotNull(idStats, "id column stats should be present"); + assertTrue(idStats.nullCount().isPresent(), "id nullCount should be present"); + assertTrue(idStats.distinctCount().isPresent(), "id distinctCount should be present"); + assertTrue(idStats.min().isPresent(), "id min should be present"); + assertTrue(idStats.max().isPresent(), "id max should be present"); + assertEquals(1, idStats.min().get(), "id min should be 1"); + assertEquals(2, idStats.max().get(), "id max should be 2"); + }); + } + + @Test + public void testEstimateStatisticsWithCatalogStats_cboDisabled(@TempDir File tempDir) + throws Exception { + String path = tempDir.getAbsolutePath(); + String tblName = "stats_cbo_disabled"; + spark.sql( + String.format( + "CREATE TABLE %s (id INT, name STRING) USING delta LOCATION '%s'", tblName, path)); + spark.sql(String.format("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tblName)); + + // Inject catalog stats + CatalogStatistics catalogStats = + new CatalogStatistics( + scala.math.BigInt.apply(512L), + scala.Option.apply(scala.math.BigInt.apply(2L)), + scala.collection.immutable.Map$.MODULE$.empty()); + + CatalogTable catalogTable = injectCatalogStats(tblName, catalogStats); + + withSQLConf( + "spark.sql.cbo.enabled", + "false", + () -> { + Identifier id = Identifier.of(new String[] {"default"}, tblName); + SparkTable sparkTable = new SparkTable(id, catalogTable, Collections.emptyMap()); + + SparkScanBuilder builder = + (SparkScanBuilder) + sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); + SparkScan scan = (SparkScan) builder.build(); + Statistics stats = scan.estimateStatistics(); + + // With CBO disabled, numRows should be empty (matching V1 behavior) + assertFalse(stats.numRows().isPresent(), "numRows should be empty with CBO disabled"); + + // sizeInBytes should still come from planned files + assertTrue(stats.sizeInBytes().isPresent(), "sizeInBytes should be present"); + assertTrue(stats.sizeInBytes().getAsLong() > 0, "sizeInBytes should be positive"); + }); + } + + @Test + public void testEstimateStatisticsWithCatalogStats_planStatsEnabled(@TempDir File tempDir) + throws Exception { + String path = tempDir.getAbsolutePath(); + String tblName = "stats_plan_stats_enabled"; + spark.sql( + String.format( + "CREATE TABLE %s (id INT, name STRING) USING delta LOCATION '%s'", tblName, path)); + spark.sql(String.format("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tblName)); + + // Inject catalog stats with numRows + CatalogStatistics catalogStats = + new CatalogStatistics( + scala.math.BigInt.apply(512L), + scala.Option.apply(scala.math.BigInt.apply(2L)), + scala.collection.immutable.Map$.MODULE$.empty()); + + CatalogTable catalogTable = injectCatalogStats(tblName, catalogStats); + + // CBO disabled but planStatsEnabled=true should still surface catalog stats + withSQLConf( + "spark.sql.cbo.enabled", + "false", + () -> { + withSQLConf( + "spark.sql.cbo.planStats.enabled", + "true", + () -> { + Identifier id = Identifier.of(new String[] {"default"}, tblName); + SparkTable sparkTable = new SparkTable(id, catalogTable, Collections.emptyMap()); + + SparkScanBuilder builder = + (SparkScanBuilder) + sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); + SparkScan scan = (SparkScan) builder.build(); + Statistics stats = scan.estimateStatistics(); + + // With planStatsEnabled, numRows should come from catalog stats + assertTrue( + stats.numRows().isPresent(), "numRows should be present with planStatsEnabled"); + assertEquals(2L, stats.numRows().getAsLong(), "numRows should be 2"); + + // sizeInBytes should still come from planned files + assertTrue(stats.sizeInBytes().isPresent(), "sizeInBytes should be present"); + assertTrue(stats.sizeInBytes().getAsLong() > 0, "sizeInBytes should be positive"); + }); + }); + } + + @Test + public void testEstimateStatisticsWithoutCatalogStats(@TempDir File tempDir) throws Exception { + // Path-based table has no catalog stats + String path = tempDir.getAbsolutePath(); + String tblName = "stats_no_catalog"; + spark.sql( + String.format( + "CREATE TABLE %s (id INT, name STRING) USING delta LOCATION '%s'", tblName, path)); + spark.sql(String.format("INSERT INTO %s VALUES (1, 'a')", tblName)); + + withSQLConf( + "spark.sql.cbo.enabled", + "true", + () -> { + // Path-based table — no catalog table, no stats + Identifier id = Identifier.of(new String[] {"default"}, tblName); + SparkTable sparkTable = new SparkTable(id, path); + + SparkScanBuilder builder = + (SparkScanBuilder) + sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); + SparkScan scan = (SparkScan) builder.build(); + Statistics stats = scan.estimateStatistics(); + + // Without catalog stats, numRows should be empty + assertFalse(stats.numRows().isPresent(), "numRows should be empty for path-based table"); + assertTrue(stats.sizeInBytes().isPresent(), "sizeInBytes should be present"); + assertTrue(stats.sizeInBytes().getAsLong() > 0, "sizeInBytes should be positive"); + }); + } + + @Test + public void testEstimateStatisticsWithPartitionedTableAndCatalogStats(@TempDir File tempDir) + throws Exception { + String path = tempDir.getAbsolutePath(); + String tblName = "stats_partitioned"; + spark.sql( + String.format( + "CREATE TABLE %s (id INT, name STRING, part INT) USING delta " + + "PARTITIONED BY (part) LOCATION '%s'", + tblName, path)); + spark.sql( + String.format("INSERT INTO %s VALUES (1, 'a', 1), (2, 'b', 1), (3, 'c', 2)", tblName)); + + // Inject catalog stats with column stats for both data and partition columns + CatalogColumnStat idColStat = + new CatalogColumnStat( + scala.Option.apply(scala.math.BigInt.apply(3L)), + scala.Option.apply("1"), + scala.Option.apply("3"), + scala.Option.apply(scala.math.BigInt.apply(0L)), + scala.Option.empty(), + scala.Option.empty(), + scala.Option.empty(), + CatalogColumnStat.VERSION()); + + CatalogColumnStat partColStat = + new CatalogColumnStat( + scala.Option.apply(scala.math.BigInt.apply(2L)), + scala.Option.apply("1"), + scala.Option.apply("2"), + scala.Option.apply(scala.math.BigInt.apply(0L)), + scala.Option.empty(), + scala.Option.empty(), + scala.Option.empty(), + CatalogColumnStat.VERSION()); + + CatalogStatistics catalogStats = + new CatalogStatistics( + scala.math.BigInt.apply(2048L), + scala.Option.apply(scala.math.BigInt.apply(3L)), + buildColStatsMap( + new String[] {"id", "part"}, new CatalogColumnStat[] {idColStat, partColStat})); + + CatalogTable catalogTable = injectCatalogStats(tblName, catalogStats); + + withSQLConf( + "spark.sql.cbo.enabled", + "true", + () -> { + Identifier id = Identifier.of(new String[] {"default"}, tblName); + SparkTable sparkTable = new SparkTable(id, catalogTable, Collections.emptyMap()); + + SparkScanBuilder builder = + (SparkScanBuilder) + sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); + SparkScan scan = (SparkScan) builder.build(); + Statistics stats = scan.estimateStatistics(); + + assertTrue(stats.numRows().isPresent(), "numRows should be present"); + assertEquals(3L, stats.numRows().getAsLong(), "numRows should be 3"); + + // Verify column stats include both data and partition columns + Map colStats = stats.columnStats(); + assertNotNull(colStats.get(FieldReference.apply("id")), "id stats should be present"); + assertNotNull(colStats.get(FieldReference.apply("part")), "part stats should be present"); + + // Check partition column stats + ColumnStatistics partStats = colStats.get(FieldReference.apply("part")); + assertTrue(partStats.min().isPresent(), "part min should be present"); + assertTrue(partStats.max().isPresent(), "part max should be present"); + assertEquals(1, partStats.min().get(), "part min should be 1"); + assertEquals(2, partStats.max().get(), "part max should be 2"); + }); + } + + @Test + public void testEstimatedSizeUsesAvgLenFromCatalogStats(@TempDir File tempDir) throws Exception { + // Verify that computeEstimatedSizeWithColumnProjection uses avgLen from catalog stats + // instead of defaultSize(), mirroring EstimationUtils.getSizePerRow() (#5952). + String path = tempDir.getAbsolutePath(); + String tblName = "stats_avglen"; + spark.sql( + String.format( + "CREATE TABLE %s (id INT, name STRING) USING delta LOCATION '%s'", tblName, path)); + spark.sql(String.format("INSERT INTO %s VALUES (1, 'a'), (2, 'bb'), (3, 'ccc')", tblName)); + + // Create column stats with avgLen=5 for 'name' (STRING defaultSize is 20) + CatalogColumnStat nameColStat = + new CatalogColumnStat( + scala.Option.empty(), + scala.Option.empty(), + scala.Option.empty(), + scala.Option.empty(), + scala.Option.apply((Object) 5L), // avgLen = 5 (vs STRING defaultSize 20) + scala.Option.empty(), + scala.Option.empty(), + CatalogColumnStat.VERSION()); + + CatalogStatistics catalogStats = + new CatalogStatistics( + scala.math.BigInt.apply(1024L), + scala.Option.apply(scala.math.BigInt.apply(3L)), + buildColStatsMap(new String[] {"name"}, new CatalogColumnStat[] {nameColStat})); + + CatalogTable catalogTable = injectCatalogStats(tblName, catalogStats); + + // avgLen is used for sizeInBytes estimation regardless of CBO/planStats settings + withSQLConf( + "spark.sql.cbo.enabled", + "false", + () -> { + withSQLConf( + "spark.sql.cbo.planStats.enabled", + "false", + () -> { + Identifier id = Identifier.of(new String[] {"default"}, tblName); + SparkTable sparkTable = new SparkTable(id, catalogTable, Collections.emptyMap()); + + SparkScanBuilder builder = + (SparkScanBuilder) + sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); + + // Prune to only 'name' column to trigger column projection estimation + StructType prunedSchema = new StructType().add("name", DataTypes.StringType); + builder.pruneColumns(prunedSchema); + + SparkScan scan = (SparkScan) builder.build(); + + long totalBytes = getTotalBytes(scan); + long estimatedSize = getEstimatedSizeInBytes(scan); + assertTrue(totalBytes > 0, "totalBytes should be positive"); + + // Schema: dataSchema = (id INT, name STRING), partitionSchema = empty + // readSchema = (name STRING) after pruning + // + // With avgLen=5 for name (STRING: avgLen + 12 = 17, vs defaultSize 20): + // fullSchemaRowSize = 8 + (4 [INT default] + 17 [STRING avgLen]) = 29 + // outputRowSize = 8 + 17 = 25 + // estimated = (totalBytes * 25) / 29 + // + // Without avgLen (defaultSize only): + // fullSchemaRowSize = 8 + (4 + 20) = 32 + // outputRowSize = 8 + 20 = 28 + // estimated = (totalBytes * 28) / 32 + long expectedWithAvgLen = (totalBytes * 25) / 29; + long expectedWithoutAvgLen = (totalBytes * 28) / 32; + + assertEquals( + expectedWithAvgLen, + estimatedSize, + "estimatedSize should use avgLen from catalog stats"); + assertNotEquals( + expectedWithoutAvgLen, + estimatedSize, + "estimatedSize should differ from defaultSize-only calculation"); + }); + }); + } + + @Test + public void testEstimateStatisticsWithoutAnalyze(@TempDir File tempDir) throws Exception { + // Table exists in catalog but no stats were injected + String path = tempDir.getAbsolutePath(); + String tblName = "stats_no_analyze"; + spark.sql( + String.format( + "CREATE TABLE %s (id INT, name STRING) USING delta LOCATION '%s'", tblName, path)); + spark.sql(String.format("INSERT INTO %s VALUES (1, 'a')", tblName)); + + withSQLConf( + "spark.sql.cbo.enabled", + "true", + () -> { + CatalogTable catalogTable = + spark.sessionState().catalog().getTableMetadata(new TableIdentifier(tblName)); + Identifier id = Identifier.of(new String[] {"default"}, tblName); + SparkTable sparkTable = new SparkTable(id, catalogTable, Collections.emptyMap()); + + SparkScanBuilder builder = + (SparkScanBuilder) + sparkTable.newScanBuilder(new CaseInsensitiveStringMap(new HashMap<>())); + SparkScan scan = (SparkScan) builder.build(); + Statistics stats = scan.estimateStatistics(); + + // Without catalog stats, we fall back to file-only stats + assertFalse(stats.numRows().isPresent(), "numRows should be empty without catalog stats"); + assertTrue(stats.sizeInBytes().isPresent(), "sizeInBytes should be present"); + assertTrue(stats.sizeInBytes().getAsLong() > 0, "sizeInBytes should be positive"); + }); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private static scala.collection.immutable.Map buildColStatsMap( + String[] keys, CatalogColumnStat[] values) { + scala.collection.mutable.Builder b = scala.collection.immutable.Map$.MODULE$.newBuilder(); + for (int i = 0; i < keys.length; i++) { + b.$plus$eq(new scala.Tuple2<>(keys[i], values[i])); + } + return (scala.collection.immutable.Map) b.result(); + } } diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/deletionvector/ColumnVectorWithFilterTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/deletionvector/ColumnVectorWithFilterTest.java new file mode 100644 index 00000000000..552e827296a --- /dev/null +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/deletionvector/ColumnVectorWithFilterTest.java @@ -0,0 +1,184 @@ +/* + * Copyright (2026) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.spark.internal.v2.read.deletionvector; + +import static org.junit.jupiter.api.Assertions.*; + +import java.util.function.IntFunction; +import org.apache.spark.sql.execution.vectorized.OnHeapColumnVector; +import org.apache.spark.sql.execution.vectorized.WritableColumnVector; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.StructType; +import org.apache.spark.sql.vectorized.ColumnVector; +import org.junit.jupiter.api.Test; + +public class ColumnVectorWithFilterTest { + + @Test + void testIntegerColumnVector() { + try (WritableColumnVector delegate = new OnHeapColumnVector(5, DataTypes.IntegerType)) { + for (int i = 0; i < 5; i++) { + delegate.putInt(i, (i + 1) * 10); // [10, 20, 30, 40, 50] + } + ColumnVectorWithFilter mappedVector = new ColumnVectorWithFilter(delegate, new int[] {1, 3}); + + assertMappedValues(mappedVector, mappedVector::getInt, new Integer[] {20, 40}); + } + } + + @Test + void testLongColumnVector() { + try (WritableColumnVector delegate = new OnHeapColumnVector(4, DataTypes.LongType)) { + for (int i = 0; i < 4; i++) { + delegate.putLong(i, (i + 1) * 100L); + } + ColumnVectorWithFilter mappedVector = new ColumnVectorWithFilter(delegate, new int[] {0, 2}); + + assertMappedValues(mappedVector, mappedVector::getLong, new Long[] {100L, 300L}); + } + } + + @Test + void testDoubleColumnVector() { + try (WritableColumnVector delegate = new OnHeapColumnVector(3, DataTypes.DoubleType)) { + delegate.putDouble(0, 1.1); + delegate.putDouble(1, 2.2); + delegate.putDouble(2, 3.3); + ColumnVectorWithFilter mappedVector = new ColumnVectorWithFilter(delegate, new int[] {0, 2}); + + assertMappedValues(mappedVector, mappedVector::getDouble, new Double[] {1.1, 3.3}); + } + } + + @Test + void testBooleanColumnVector() { + try (WritableColumnVector delegate = new OnHeapColumnVector(4, DataTypes.BooleanType)) { + delegate.putBoolean(0, true); + delegate.putBoolean(1, false); + delegate.putBoolean(2, true); + delegate.putBoolean(3, false); + ColumnVectorWithFilter mappedVector = new ColumnVectorWithFilter(delegate, new int[] {1, 2}); + + assertMappedValues(mappedVector, mappedVector::getBoolean, new Boolean[] {false, true}); + } + } + + @Test + void testStringColumnVector() { + try (WritableColumnVector delegate = new OnHeapColumnVector(3, DataTypes.StringType)) { + delegate.putByteArray(0, "alice".getBytes()); + delegate.putByteArray(1, "bob".getBytes()); + delegate.putByteArray(2, "charlie".getBytes()); + ColumnVectorWithFilter mappedVector = new ColumnVectorWithFilter(delegate, new int[] {2}); + + assertMappedValues( + mappedVector, i -> mappedVector.getUTF8String(i).toString(), new String[] {"charlie"}); + } + } + + @Test + void testStructColumnVector() { + StructType structType = + new StructType().add("id", DataTypes.IntegerType).add("name", DataTypes.StringType); + try (WritableColumnVector delegate = new OnHeapColumnVector(3, structType)) { + WritableColumnVector idChild = (WritableColumnVector) delegate.getChild(0); + for (int i = 0; i < 3; i++) { + idChild.putInt(i, i + 1); // [1, 2, 3] + } + WritableColumnVector nameChild = (WritableColumnVector) delegate.getChild(1); + nameChild.putByteArray(0, "a".getBytes()); + nameChild.putByteArray(1, "b".getBytes()); + nameChild.putByteArray(2, "c".getBytes()); + + ColumnVectorWithFilter mappedVector = new ColumnVectorWithFilter(delegate, new int[] {0, 2}); + + ColumnVector mappedIdColumn = mappedVector.getChild(0); + ColumnVector mappedNameColumn = mappedVector.getChild(1); + + assertMappedValues(mappedIdColumn, mappedIdColumn::getInt, new Integer[] {1, 3}); + assertMappedValues( + mappedNameColumn, + i -> mappedNameColumn.getUTF8String(i).toString(), + new String[] {"a", "c"}); + } + } + + @Test + void testNullColumnVector() { + try (WritableColumnVector delegate = new OnHeapColumnVector(4, DataTypes.IntegerType)) { + // original rows: [10, null, 30, null] + delegate.putInt(0, 10); + delegate.putNull(1); + delegate.putInt(2, 30); + delegate.putNull(3); + // selected rows (in order): [3, 0, 1] => [null, 10, null] + ColumnVectorWithFilter mappedVector = + new ColumnVectorWithFilter(delegate, new int[] {3, 0, 1}); + + assertMappedValues(mappedVector, mappedVector::getInt, new Integer[] {null, 10, null}); + } + } + + @Test + void testAllNullColumnVector() { + try (WritableColumnVector delegate = new OnHeapColumnVector(3, DataTypes.IntegerType)) { + delegate.putNull(0); + delegate.putNull(1); + delegate.putNull(2); + ColumnVectorWithFilter mappedVector = new ColumnVectorWithFilter(delegate, new int[] {2, 0}); + + assertMappedValues(mappedVector, mappedVector::getInt, new Integer[] {null, null}); + } + } + + @Test + void testNoneSelectColumnVector() { + try (WritableColumnVector delegate = new OnHeapColumnVector(3, DataTypes.IntegerType)) { + delegate.putInt(0, 10); + delegate.putInt(1, 20); + delegate.putInt(2, 30); + + ColumnVectorWithFilter noRowsSelected = new ColumnVectorWithFilter(delegate, new int[] {}); + assertEquals(DataTypes.IntegerType, noRowsSelected.dataType()); + } + } + + @Test + void testAllSelectColumnVector() { + try (WritableColumnVector delegate = new OnHeapColumnVector(3, DataTypes.IntegerType)) { + delegate.putInt(0, 10); + delegate.putInt(1, 20); + delegate.putInt(2, 30); + + // all select + ColumnVectorWithFilter allRowsSelected = + new ColumnVectorWithFilter(delegate, new int[] {0, 1, 2}); + assertMappedValues(allRowsSelected, allRowsSelected::getInt, new Integer[] {10, 20, 30}); + } + } + + private static void assertMappedValues( + ColumnVector vector, IntFunction getter, T[] expected) { + for (int i = 0; i < expected.length; i++) { + if (expected[i] == null) { + assertTrue(vector.isNullAt(i), "Expected null at mapped row " + i); + } else { + assertFalse(vector.isNullAt(i), "Expected non-null at mapped row " + i); + assertEquals(expected[i], getter.apply(i), "Mismatch at mapped row " + i); + } + } + } +} diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/deletionvector/DeletionVectorReadFunctionTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/deletionvector/DeletionVectorReadFunctionTest.java index 1747dc96da0..5c914ac4d2f 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/read/deletionvector/DeletionVectorReadFunctionTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/read/deletionvector/DeletionVectorReadFunctionTest.java @@ -16,11 +16,17 @@ package io.delta.spark.internal.v2.read.deletionvector; import static io.delta.spark.internal.v2.InternalRowTestUtils.*; +import static org.junit.jupiter.api.Assertions.*; +import java.nio.charset.StandardCharsets; import java.util.List; import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.execution.vectorized.OnHeapColumnVector; +import org.apache.spark.sql.execution.vectorized.WritableColumnVector; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructType; +import org.apache.spark.sql.vectorized.ColumnVector; +import org.apache.spark.sql.vectorized.ColumnarBatch; import org.junit.jupiter.api.Test; public class DeletionVectorReadFunctionTest { @@ -29,6 +35,8 @@ public class DeletionVectorReadFunctionTest { new StructType().add("id", DataTypes.IntegerType).add("name", DataTypes.StringType); private static final StructType PARTITION_SCHEMA = new StructType(); + // ===== Row-based tests ===== + @Test public void testFilterDeletedRowsAndProjectRemovesDvColumn() { // Input: 3 rows, middle one is deleted. @@ -41,9 +49,9 @@ public void testFilterDeletedRowsAndProjectRemovesDvColumn() { DeletionVectorSchemaContext context = new DeletionVectorSchemaContext(DATA_SCHEMA, PARTITION_SCHEMA); DeletionVectorReadFunction readFunc = - DeletionVectorReadFunction.wrap(mockReader(inputRows), context); + DeletionVectorReadFunction.wrap(mockReader(inputRows), context, false); - List result = collectRows(readFunc.apply(null)); + List result = collectRows(readFunc.apply(/* file= */ null)); // Verify filtered and projected output (DV column removed, deleted row filtered). assertRowsEquals(result, List.of(row(1, "alice"), row(3, "charlie"))); @@ -57,9 +65,9 @@ public void testAllRowsDeleted() { DeletionVectorSchemaContext context = new DeletionVectorSchemaContext(DATA_SCHEMA, PARTITION_SCHEMA); DeletionVectorReadFunction readFunc = - DeletionVectorReadFunction.wrap(mockReader(inputRows), context); + DeletionVectorReadFunction.wrap(mockReader(inputRows), context, false); - List result = collectRows(readFunc.apply(null)); + List result = collectRows(readFunc.apply(/* file= */ null)); assertRowsEquals(result, List.of()); } @@ -72,10 +80,101 @@ public void testNoRowsDeleted() { DeletionVectorSchemaContext context = new DeletionVectorSchemaContext(DATA_SCHEMA, PARTITION_SCHEMA); DeletionVectorReadFunction readFunc = - DeletionVectorReadFunction.wrap(mockReader(inputRows), context); + DeletionVectorReadFunction.wrap(mockReader(inputRows), context, false); - List result = collectRows(readFunc.apply(null)); + List result = collectRows(readFunc.apply(/* file= */ null)); assertRowsEquals(result, List.of(row(1, "alice"), row(2, "bob"), row(3, "charlie"))); } + + // ===== ColumnarBatch (vectorized) tests ===== + + @Test + public void testBatchFilterDeletedRowsAndProjectRemovesDvColumn() { + // 3 rows: row 1 deleted, rows 0 and 2 kept. + ColumnarBatch inputBatch = createBatch(new int[] {1, 2, 3}, new byte[] {0, 1, 0}); + List result = runBatchRead(inputBatch); + + assertEquals(1, result.size()); + // Filtered: row 0 -> original 0 (id=1), row 1 -> original 2 (id=3) + assertBatchRows(result.get(0), new int[] {1, 3}, new String[] {"name_0", "name_2"}); + } + + @Test + public void testBatchAllRowsDeleted() { + ColumnarBatch inputBatch = createBatch(new int[] {1, 2}, new byte[] {1, 1}); + List result = runBatchRead(inputBatch); + + assertEquals(1, result.size()); + assertBatchRows(result.get(0), new int[] {}, new String[] {}); + } + + @Test + public void testBatchNoRowsDeleted() { + ColumnarBatch inputBatch = createBatch(new int[] {1, 2, 3}, new byte[] {0, 0, 0}); + List result = runBatchRead(inputBatch); + + assertEquals(1, result.size()); + assertBatchRows( + result.get(0), new int[] {1, 2, 3}, new String[] {"name_0", "name_1", "name_2"}); + } + + @Test + public void testBatchMultipleBatchesWithDifferentBatch() { + ColumnarBatch allDeleted = createBatch(new int[] {1, 2, 3}, new byte[] {1, 1, 1}); + ColumnarBatch mixed = createBatch(new int[] {4, 5, 6}, new byte[] {0, 1, 0}); + ColumnarBatch allLive = createBatch(new int[] {7}, new byte[] {0}); + + // Ordering 1: allDeleted, mixed, allLive + List result1 = runBatchRead(allDeleted, mixed, allLive); + assertEquals(3, result1.size()); + assertBatchRows(result1.get(0), new int[] {}, new String[] {}); + assertBatchRows(result1.get(1), new int[] {4, 6}, new String[] {"name_0", "name_2"}); + assertBatchRows(result1.get(2), new int[] {7}, new String[] {"name_0"}); + + // Ordering 2: allLive, allDeleted, mixed + List result2 = runBatchRead(allLive, allDeleted, mixed); + assertEquals(3, result2.size()); + assertBatchRows(result2.get(0), new int[] {7}, new String[] {"name_0"}); + assertBatchRows(result2.get(1), new int[] {}, new String[] {}); + assertBatchRows(result2.get(2), new int[] {4, 6}, new String[] {"name_0", "name_2"}); + } + + private List runBatchRead(ColumnarBatch... inputBatches) { + DeletionVectorSchemaContext context = + new DeletionVectorSchemaContext(DATA_SCHEMA, PARTITION_SCHEMA); + DeletionVectorReadFunction readFunc = + DeletionVectorReadFunction.wrap(mockBatchReader(List.of(inputBatches)), context, true); + return collectBatches(readFunc.apply(/* file= */ null)); + } + + private static void assertBatchRows( + ColumnarBatch batch, int[] expectedIds, String[] expectedNames) { + assertEquals(expectedIds.length, expectedNames.length, "Expected id/name lengths must match"); + assertEquals(expectedIds.length, batch.numRows(), "Unexpected number of filtered rows"); + for (int i = 0; i < expectedIds.length; i++) { + assertEquals(expectedIds[i], batch.column(0).getInt(i)); + assertEquals(expectedNames[i], batch.column(1).getUTF8String(i).toString()); + } + } + + /** + * Creates a ColumnarBatch with columns [id (int), name (string), is_row_deleted (byte)]. + * + *

    Name values are auto-generated as "name_0", "name_1", etc. + */ + private static ColumnarBatch createBatch(int[] ids, byte[] deletionVector) { + int numRows = ids.length; + WritableColumnVector idCol = new OnHeapColumnVector(numRows, DataTypes.IntegerType); + WritableColumnVector nameCol = new OnHeapColumnVector(numRows, DataTypes.StringType); + WritableColumnVector dvCol = new OnHeapColumnVector(numRows, DataTypes.ByteType); + + for (int i = 0; i < numRows; i++) { + idCol.putInt(i, ids[i]); + nameCol.putByteArray(i, ("name_" + i).getBytes(StandardCharsets.UTF_8)); + dvCol.putByte(i, deletionVector[i]); + } + + return new ColumnarBatch(new ColumnVector[] {idCol, nameCol, dvCol}, numRows); + } } diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/ExpressionUtilsTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/ExpressionUtilsTest.java index f80d236c3c1..b9f6d46d6b3 100644 --- a/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/ExpressionUtilsTest.java +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/ExpressionUtilsTest.java @@ -22,7 +22,9 @@ import io.delta.kernel.expressions.Predicate; import io.delta.kernel.types.*; import java.math.BigDecimal; +import java.util.HashSet; import java.util.Optional; +import java.util.Set; import java.util.function.Supplier; import java.util.stream.Stream; import org.apache.spark.sql.connector.expressions.FieldReference; @@ -109,6 +111,136 @@ public void testEqualNullSafeFilter() { assertEquals(2, nonNullResult.get().getChildren().size()); } + static Stream stringStartsWithProvider() { + return Stream.of(Arguments.of("non-empty prefix", "Al"), Arguments.of("empty prefix", "")); + } + + @ParameterizedTest(name = "StringStartsWith with {0} should be converted") + @MethodSource("stringStartsWithProvider") + public void testStringStartsWithFilter(String desc, String prefix) { + StringStartsWith filter = new StringStartsWith("name", prefix); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter); + + assertTrue(result.isPresent(), "StringStartsWith filter should be converted"); + assertFalse(result.isPartial(), "StringStartsWith filter should be fully converted"); + assertEquals("STARTS_WITH", result.get().getName()); + // Children: column + string literal + assertEquals(2, result.get().getChildren().size()); + } + + @Test + public void testStringStartsWithFilter_NullValue() { + // A null prefix cannot be converted — treated as unsupported, falls back to post-scan + StringStartsWith filter = new StringStartsWith("name", null); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter); + + assertFalse(result.isPresent(), "StringStartsWith with null value should not be converted"); + } + + @Test + public void testInFilter_BasicConversion() { + In filter = new In("city", new Object[] {"hz", "sh", "bj"}); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter); + + assertTrue(result.isPresent(), "In filter should be converted"); + assertFalse(result.isPartial(), "In filter should be fully converted"); + assertEquals("IN", result.get().getName()); + // Children: column + 3 literals + assertEquals(4, result.get().getChildren().size()); + } + + @Test + public void testInFilter_SingleValue() { + In filter = new In("id", new Object[] {42}); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter); + + assertTrue(result.isPresent(), "In filter with single value should be converted"); + assertFalse(result.isPartial(), "In filter should be fully converted"); + assertEquals("IN", result.get().getName()); + // Children: column + 1 literal + assertEquals(2, result.get().getChildren().size()); + } + + @Test + public void testInFilter_EmptyValues() { + In filter = new In("city", new Object[] {}); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter); + + // Empty IN list always evaluates to FALSE; push ALWAYS_FALSE so the kernel skips all files. + assertTrue(result.isPresent(), "In filter with empty values should push ALWAYS_FALSE"); + assertEquals("ALWAYS_FALSE", result.get().getName()); + } + + @Test + public void testInFilter_WithNullValue() { + // null in the values array makes the IN expression unsafe to push down (SQL null semantics) + In filter = new In("city", new Object[] {"hz", null, "bj"}); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter); + + assertFalse(result.isPresent(), "In filter with null value should not be pushed down"); + } + + @Test + public void testInFilter_WithUnsupportedType() { + In filter = new In("col", new Object[] {42, new Object()}); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter); + + assertFalse(result.isPresent(), "In filter with unconvertible value should not be pushed down"); + } + + @Test + public void testInFilter_InAndFilter() { + // AND(In(...), EqualTo(...)) — both convertible, should be fully pushed down + In inFilter = new In("city", new Object[] {"hz", "sh"}); + EqualTo eqFilter = new EqualTo("part", 1); + org.apache.spark.sql.sources.And andFilter = + new org.apache.spark.sql.sources.And(inFilter, eqFilter); + + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(andFilter); + + assertTrue(result.isPresent(), "AND(In, EqualTo) should be converted"); + assertFalse(result.isPartial(), "AND(In, EqualTo) should be fully converted"); + assertTrue( + result.get() instanceof io.delta.kernel.expressions.And, + "Result should be an AND predicate"); + } + + @Test + public void testInFilter_NotInWithNullValue() { + // NOT(IN(col, 1, null)): null in the IN list causes IN to bail → NOT also bails + In inFilter = new In("city", new Object[] {"hz", null}); + Not notFilter = new Not(inFilter); + + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(notFilter); + + assertFalse( + result.isPresent(), "NOT(IN(..., null)) should not be pushed down due to null in IN list"); + } + + @Test + public void testInFilter_ORWithUnsupportedFilter() { + // OR(In(...), StringEndsWith(...)) — one branch unsupported, whole OR cannot be pushed + In inFilter = new In("city", new Object[] {"hz", "sh"}); + StringEndsWith endsWithFilter = new StringEndsWith("name", "foo"); + Or orFilter = new Or(inFilter, endsWithFilter); + + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(orFilter); + + assertFalse( + result.isPresent(), + "OR(In, unsupported) should not be pushed down when one branch is unsupported"); + } + // Test data provider for parameterized literal conversion tests static Stream valueTypesProvider() { return Stream.of( @@ -139,6 +271,28 @@ static Stream valueTypesProvider() { TimestampType.TIMESTAMP)); } + @Test + public void testAlwaysTrueFilter() { + Filter filter = new org.apache.spark.sql.sources.AlwaysTrue(); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter); + + assertTrue(result.isPresent(), "AlwaysTrue should be converted"); + assertFalse(result.isPartial()); + assertEquals("ALWAYS_TRUE", result.get().getName()); + } + + @Test + public void testAlwaysFalseFilter() { + Filter filter = new org.apache.spark.sql.sources.AlwaysFalse(); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter); + + assertTrue(result.isPresent(), "AlwaysFalse should be converted"); + assertFalse(result.isPartial()); + assertEquals("ALWAYS_FALSE", result.get().getName()); + } + @Test public void testUnsupportedFilter() { // Create an unsupported filter (StringContains is not implemented in our conversion method) @@ -319,6 +473,22 @@ public void testConvertValueToKernelLiteral( "DataType should match expected type for " + typeName); } + @Test + public void testConvertValueToKernelLiteral_DecimalWithScaleExceedingPrecision() { + // BigDecimal("0.00") has precision=1, scale=2 which violates precision >= scale. + // The conversion should normalize precision to max(precision, scale + 1). + BigDecimal bd = new BigDecimal("0.00"); + assertEquals(1, bd.precision(), "Precondition: precision of 0.00 should be 1"); + assertEquals(2, bd.scale(), "Precondition: scale of 0.00 should be 2"); + + Optional result = ExpressionUtils.convertValueToKernelLiteral(bd); + assertTrue(result.isPresent(), "BigDecimal 0.00 should be convertible"); + + Literal literal = result.get(); + // Normalized precision should be max(1, 2+1) = 3 + assertEquals(new DecimalType(3, 2), literal.getDataType()); + } + @Test public void testConvertValueToKernelLiteral_NullValue() { Optional result = ExpressionUtils.convertValueToKernelLiteral(null); @@ -601,6 +771,261 @@ public void testDsv2PredicateToCatalystExpression_UnsupportedPredicate() { assertFalse(result.isPresent(), "Unsupported predicates should return empty Optional"); } + // ===== Tests for decimal type alignment with table schema ===== + + private final org.apache.spark.sql.types.StructType decimalSchema = + new org.apache.spark.sql.types.StructType() + .add("price", new org.apache.spark.sql.types.DecimalType(7, 2), true) + .add("quantity", org.apache.spark.sql.types.DataTypes.IntegerType, true); + + @Test + public void testDecimalLiteralWidenedToColumnType() { + // Literal 100.00 has Decimal(5,2), column is Decimal(7,2) → should widen to Decimal(7,2) + EqualTo filter = new EqualTo("price", new BigDecimal("100.00")); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter, decimalSchema); + + assertTrue(result.isPresent(), "Decimal filter should be converted with widened type"); + assertFalse(result.isPartial()); + Predicate pred = result.get(); + assertEquals("=", pred.getName()); + Literal literal = (Literal) pred.getChildren().get(1); + assertEquals(new DecimalType(7, 2), literal.getDataType()); + } + + @Test + public void testDecimalLiteralScaleWidened() { + // Literal 100 has Decimal(3,0), column is Decimal(7,2) → should widen to Decimal(7,2) + EqualTo filter = new EqualTo("price", new BigDecimal("100")); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter, decimalSchema); + + assertTrue(result.isPresent(), "Decimal filter with lower scale should be widened"); + Literal literal = (Literal) result.get().getChildren().get(1); + assertEquals(new DecimalType(7, 2), literal.getDataType()); + assertEquals(new BigDecimal("100.00"), literal.getValue()); + } + + @Test + public void testDecimalLiteralHigherScaleThanColumn() { + // Literal 99.999 has scale=3, column is Decimal(7,2) → scale exceeds column, skip pushdown + GreaterThan filter = new GreaterThan("price", new BigDecimal("99.999")); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter, decimalSchema); + + assertFalse( + result.isPresent(), + "Decimal literal with higher scale than column should not be pushed down"); + } + + @Test + public void testDecimalLiteralExceedsColumnPrecision() { + // Literal 123456.00 has 6 integral digits + 2 scale = precision 8, + // column is Decimal(7,2) which holds max 99999.99 → skip pushdown + LessThan filter = new LessThan("price", new BigDecimal("123456.00")); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter, decimalSchema); + + assertFalse( + result.isPresent(), "Decimal literal exceeding column precision should not be pushed down"); + } + + @Test + public void testDecimalLiteralMatchingType() { + // Literal already matches column type Decimal(7,2) → no widening needed + EqualTo filter = new EqualTo("price", new BigDecimal("12345.67")); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter, decimalSchema); + + assertTrue(result.isPresent(), "Decimal filter with matching type should be converted"); + Literal literal = (Literal) result.get().getChildren().get(1); + assertEquals(new DecimalType(7, 2), literal.getDataType()); + } + + @Test + public void testDecimalLiteralWithoutSchema() { + // Without schema, decimal literal retains its intrinsic type + EqualTo filter = new EqualTo("price", new BigDecimal("100.00")); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter); + + assertTrue(result.isPresent(), "Decimal filter without schema should still be converted"); + Literal literal = (Literal) result.get().getChildren().get(1); + // 100.00 has precision=5, scale=2 + assertEquals(new DecimalType(5, 2), literal.getDataType()); + } + + @Test + public void testDecimalLiteralNonDecimalColumn() { + // Column "quantity" is IntegerType, not DecimalType → use default conversion + EqualTo filter = new EqualTo("quantity", new BigDecimal("42")); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter, decimalSchema); + + assertTrue(result.isPresent(), "BigDecimal for non-decimal column should use default type"); + Literal literal = (Literal) result.get().getChildren().get(1); + // Default BigDecimal conversion: precision=2, scale=0 + assertEquals(new DecimalType(2, 0), literal.getDataType()); + } + + @Test + public void testDecimalLiteralCaseInsensitiveColumnLookup() { + // Filter uses "PRICE" but schema has "price" → should still widen + EqualTo filter = new EqualTo("PRICE", new BigDecimal("100.00")); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter, decimalSchema); + + assertTrue(result.isPresent(), "Case-insensitive column lookup should match"); + Literal literal = (Literal) result.get().getChildren().get(1); + assertEquals(new DecimalType(7, 2), literal.getDataType()); + } + + @Test + public void testDecimalLiteralColumnNotInSchema() { + // Column "unknown" not in schema → use default conversion + EqualTo filter = new EqualTo("unknown", new BigDecimal("100.00")); + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter, decimalSchema); + + assertTrue(result.isPresent(), "Unknown column should fall back to default conversion"); + Literal literal = (Literal) result.get().getChildren().get(1); + assertEquals(new DecimalType(5, 2), literal.getDataType()); + } + + @Test + public void testDecimalPartialPushDownInAndFilter() { + // AND(price > 99.999, price < 200.00) where left has scale=3 exceeding column's scale=2. + // Only the right side should be pushed down (partial pushdown). + GreaterThan left = new GreaterThan("price", new BigDecimal("99.999")); + LessThan right = new LessThan("price", new BigDecimal("200.00")); + org.apache.spark.sql.sources.And andFilter = new org.apache.spark.sql.sources.And(left, right); + + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(andFilter, decimalSchema); + + assertTrue(result.isPresent(), "AND filter should partially push down the valid side"); + assertTrue(result.isPartial(), "Result should be marked as partial conversion"); + // Only the right side (price < 200.00) should be pushed, not a compound AND + Predicate pred = result.get(); + assertNotEquals( + "AND", + pred.getName(), + "Left side (price > 99.999) should be dropped, not pushed as compound AND"); + assertEquals("<", pred.getName()); + Literal literal = (Literal) pred.getChildren().get(1); + assertEquals(new DecimalType(7, 2), literal.getDataType()); + assertEquals(new BigDecimal("200.00"), literal.getValue()); + } + + @Test + public void testDecimalLiteralInCompoundFilter() { + // AND(price >= 100.00, price <= 200.00) with Decimal(7,2) column + GreaterThanOrEqual left = new GreaterThanOrEqual("price", new BigDecimal("100.00")); + LessThanOrEqual right = new LessThanOrEqual("price", new BigDecimal("200.00")); + org.apache.spark.sql.sources.And andFilter = new org.apache.spark.sql.sources.And(left, right); + + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(andFilter, decimalSchema); + + assertTrue(result.isPresent(), "AND filter with decimal operands should be converted"); + assertFalse(result.isPartial()); + // Verify both literals are widened to Decimal(7,2) + io.delta.kernel.expressions.And andPred = (io.delta.kernel.expressions.And) result.get(); + Predicate leftPred = (Predicate) andPred.getChildren().get(0); + Predicate rightPred = (Predicate) andPred.getChildren().get(1); + assertEquals(new DecimalType(7, 2), ((Literal) leftPred.getChildren().get(1)).getDataType()); + assertEquals(new DecimalType(7, 2), ((Literal) rightPred.getChildren().get(1)).getDataType()); + } + + @Test + public void testDecimalLiteralInOrFilter() { + // OR(price >= 100.00, price >= 200.00) with Decimal(7,2) column + GreaterThanOrEqual left = new GreaterThanOrEqual("price", new BigDecimal("100.00")); + GreaterThanOrEqual right = new GreaterThanOrEqual("price", new BigDecimal("200.00")); + org.apache.spark.sql.sources.Or orFilter = new org.apache.spark.sql.sources.Or(left, right); + + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(orFilter, decimalSchema); + + assertTrue(result.isPresent(), "OR filter with decimal operands should be converted"); + } + + @Test + public void testDecimalLiteralInNotFilter() { + // NOT(price = 100.00) with Decimal(7,2) column + EqualTo eq = new EqualTo("price", new BigDecimal("100.00")); + Not notFilter = new Not(eq); + + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(notFilter, decimalSchema); + + assertTrue(result.isPresent(), "NOT filter with decimal operand should be converted"); + Predicate notPred = result.get(); + assertEquals("NOT", notPred.getName()); + Predicate innerPred = (Predicate) notPred.getChildren().get(0); + Literal literal = (Literal) innerPred.getChildren().get(1); + assertEquals(new DecimalType(7, 2), literal.getDataType()); + } + + @Test + public void testDecimalLiteralWithEqualNullSafe() { + // EqualNullSafe(price, 100.00) with Decimal(7,2) column + EqualNullSafe filter = new EqualNullSafe("price", new BigDecimal("100.00")); + + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filter, decimalSchema); + + assertTrue(result.isPresent(), "EqualNullSafe with decimal should be converted"); + Literal literal = (Literal) result.get().getChildren().get(1); + assertEquals(new DecimalType(7, 2), literal.getDataType()); + } + + @Test + public void testDecimalLiteralAllComparisonOperators() { + // Test all comparison operators widen decimals correctly + BigDecimal value = new BigDecimal("50.00"); // Decimal(4,2) → should widen to Decimal(7,2) + Filter[] filters = + new Filter[] { + new EqualTo("price", value), + new GreaterThan("price", value), + new GreaterThanOrEqual("price", value), + new LessThan("price", value), + new LessThanOrEqual("price", value), + }; + String[] expectedOps = new String[] {"=", ">", ">=", "<", "<="}; + + for (int i = 0; i < filters.length; i++) { + ExpressionUtils.ConvertedPredicate result = + ExpressionUtils.convertSparkFilterToKernelPredicate(filters[i], decimalSchema); + assertTrue(result.isPresent(), expectedOps[i] + " filter should be converted"); + Literal literal = (Literal) result.get().getChildren().get(1); + assertEquals( + new DecimalType(7, 2), + literal.getDataType(), + expectedOps[i] + " should widen decimal to column type"); + } + } + + @Test + public void testClassifyFilterWithNullSchemaMatchesTwoArgOverload() { + // Directly validates that classifyFilter(filter, partitionColumns, null) produces + // the same result as the 2-arg classifyFilter(filter, partitionColumns). + Set partitionColumns = new HashSet<>(); + partitionColumns.add("dep_id"); + + EqualTo filter = new EqualTo("price", new BigDecimal("100.00")); + + ExpressionUtils.FilterClassificationResult resultTwoArg = + ExpressionUtils.classifyFilter(filter, partitionColumns); + ExpressionUtils.FilterClassificationResult resultThreeArg = + ExpressionUtils.classifyFilter(filter, partitionColumns, null); + + assertEquals(resultTwoArg.isKernelSupported, resultThreeArg.isKernelSupported); + assertEquals(resultTwoArg.isPartialConversion, resultThreeArg.isPartialConversion); + assertEquals(resultTwoArg.isDataFilter, resultThreeArg.isDataFilter); + assertEquals(resultTwoArg.kernelPredicate, resultThreeArg.kernelPredicate); + } + @Test public void testDsv2PredicateToCatalystExpression_ColumnNotFound() { NamedReference invalidRef = FieldReference.apply("nonexistent_column"); diff --git a/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/StatsUtilsTest.java b/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/StatsUtilsTest.java new file mode 100644 index 00000000000..8b35cd97fc5 --- /dev/null +++ b/spark/v2/src/test/java/io/delta/spark/internal/v2/utils/StatsUtilsTest.java @@ -0,0 +1,162 @@ +/* + * Copyright (2025) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.spark.internal.v2.utils; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Map; +import org.apache.spark.sql.catalyst.catalog.CatalogColumnStat; +import org.apache.spark.sql.catalyst.catalog.CatalogStatistics; +import org.apache.spark.sql.connector.expressions.FieldReference; +import org.apache.spark.sql.connector.expressions.NamedReference; +import org.apache.spark.sql.connector.read.Statistics; +import org.apache.spark.sql.connector.read.colstats.ColumnStatistics; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.StructType; +import org.junit.jupiter.api.Test; +import scala.Option; + +class StatsUtilsTest { + + @Test + void testToV2Statistics_sizeAndRowCount() { + StructType dataSchema = + new StructType().add("id", DataTypes.IntegerType).add("name", DataTypes.StringType); + StructType partitionSchema = new StructType(); + + CatalogStatistics catalogStats = + new CatalogStatistics( + BigInt(1024L), + Option.apply(BigInt(100L)), + scala.collection.immutable.Map$.MODULE$.empty()); + + Statistics v2Stats = StatsUtils.toV2Statistics(catalogStats, dataSchema, partitionSchema); + + assertEquals(1024L, v2Stats.sizeInBytes().getAsLong(), "sizeInBytes should match"); + assertTrue(v2Stats.numRows().isPresent(), "numRows should be present"); + assertEquals(100L, v2Stats.numRows().getAsLong(), "numRows should match"); + assertTrue(v2Stats.columnStats().isEmpty(), "columnStats should be empty"); + } + + @Test + void testToV2Statistics_sizeOnlyNoRowCount() { + StructType dataSchema = new StructType().add("id", DataTypes.IntegerType); + StructType partitionSchema = new StructType(); + + CatalogStatistics catalogStats = + new CatalogStatistics( + BigInt(512L), Option.empty(), scala.collection.immutable.Map$.MODULE$.empty()); + + Statistics v2Stats = StatsUtils.toV2Statistics(catalogStats, dataSchema, partitionSchema); + + assertEquals(512L, v2Stats.sizeInBytes().getAsLong(), "sizeInBytes should match"); + assertFalse(v2Stats.numRows().isPresent(), "numRows should be empty"); + } + + @Test + void testToV2Statistics_withColumnStats() { + StructType dataSchema = + new StructType().add("id", DataTypes.IntegerType).add("name", DataTypes.StringType); + StructType partitionSchema = new StructType().add("part", DataTypes.IntegerType); + + // Create column stats for "id" column + CatalogColumnStat idColStat = + new CatalogColumnStat( + Option.apply(BigInt(10L)), // distinctCount + Option.apply("1"), // min + Option.apply("100"), // max + Option.apply(BigInt(0L)), // nullCount + Option.apply((Object) 4L), // avgLen + Option.apply((Object) 4L), // maxLen + Option.empty(), // histogram + CatalogColumnStat.VERSION()); + + scala.collection.immutable.Map colStatsMap = + buildScalaMap(new String[] {"id"}, new CatalogColumnStat[] {idColStat}); + + CatalogStatistics catalogStats = + new CatalogStatistics(BigInt(2048L), Option.apply(BigInt(50L)), colStatsMap); + + Statistics v2Stats = StatsUtils.toV2Statistics(catalogStats, dataSchema, partitionSchema); + + assertEquals(2048L, v2Stats.sizeInBytes().getAsLong()); + assertEquals(50L, v2Stats.numRows().getAsLong()); + + Map colStats = v2Stats.columnStats(); + assertEquals(1, colStats.size(), "Should have 1 column stat"); + + ColumnStatistics idStats = colStats.get(FieldReference.apply("id")); + assertNotNull(idStats, "id column stats should be present"); + assertEquals(10L, idStats.distinctCount().getAsLong(), "distinctCount should be 10"); + assertEquals(0L, idStats.nullCount().getAsLong(), "nullCount should be 0"); + assertEquals(4L, idStats.avgLen().getAsLong(), "avgLen should be 4"); + assertEquals(4L, idStats.maxLen().getAsLong(), "maxLen should be 4"); + assertTrue(idStats.min().isPresent(), "min should be present"); + assertTrue(idStats.max().isPresent(), "max should be present"); + assertEquals(1, idStats.min().get(), "min should be 1"); + assertEquals(100, idStats.max().get(), "max should be 100"); + } + + @Test + void testToV2Statistics_skipsColumnsNotInSchema() { + // Only "id" is in schema, "unknown" should be skipped + StructType dataSchema = new StructType().add("id", DataTypes.IntegerType); + StructType partitionSchema = new StructType(); + + CatalogColumnStat colStat = + new CatalogColumnStat( + Option.apply(BigInt(5L)), + Option.empty(), + Option.empty(), + Option.apply(BigInt(1L)), + Option.empty(), + Option.empty(), + Option.empty(), + CatalogColumnStat.VERSION()); + + scala.collection.immutable.Map colStatsMap = + buildScalaMap(new String[] {"id", "unknown"}, new CatalogColumnStat[] {colStat, colStat}); + + CatalogStatistics catalogStats = + new CatalogStatistics(BigInt(100L), Option.empty(), colStatsMap); + + Statistics v2Stats = StatsUtils.toV2Statistics(catalogStats, dataSchema, partitionSchema); + + Map result = v2Stats.columnStats(); + assertEquals(1, result.size(), "Should only have 1 column stat (unknown skipped)"); + assertNotNull(result.get(FieldReference.apply("id")), "id stats should be present"); + } + + private static scala.math.BigInt BigInt(long value) { + return scala.math.BigInt.apply(value); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private static scala.collection.immutable.Map buildScalaMap( + String[] keys, CatalogColumnStat[] values) { + scala.collection.mutable.Builder< + scala.Tuple2, + scala.collection.immutable.Map> + b = (scala.collection.mutable.Builder) scala.collection.immutable.Map$.MODULE$.newBuilder(); + for (int i = 0; i < keys.length; i++) { + b.$plus$eq(new scala.Tuple2<>(keys[i], values[i])); + } + return b.result(); + } +} diff --git a/storage/src/main/java/io/delta/storage/commit/uccommitcoordinator/UCCommitCoordinatorClient.java b/storage/src/main/java/io/delta/storage/commit/uccommitcoordinator/UCCommitCoordinatorClient.java index e991b17785c..b1fd4dbf270 100644 --- a/storage/src/main/java/io/delta/storage/commit/uccommitcoordinator/UCCommitCoordinatorClient.java +++ b/storage/src/main/java/io/delta/storage/commit/uccommitcoordinator/UCCommitCoordinatorClient.java @@ -71,6 +71,12 @@ public UCCommitCoordinatorClient(Map conf, UCClient ucClient) { /** Key used to identify the write version in protocol communications with the UC server. */ private static final String WRITE_VERSION_KEY = "writeVersion"; + /** + * Temporary kill switch for sending metadata updates through UC from the Spark path. + * TODO(issue #6296): remove once metadata updates are supported end-to-end. + */ + private static final boolean SHOULD_PASS_METADATA_TO_UC = false; + // Unity Catalog Identifiers /** * Key for identifying Unity Catalog table ID in `delta.coordinatedCommits.tableConf{-preview}`. @@ -429,7 +435,7 @@ protected CommitResponse commitImpl( Optional.of(commitTimestamp), Optional.of(lastKnownBackfilledVersion.get()), disown, - updatedActions.getNewMetadata() == updatedActions.getOldMetadata() ? + updatedActions.getNewMetadata() == updatedActions.getOldMetadata() || !SHOULD_PASS_METADATA_TO_UC ? Optional.empty() : Optional.of(updatedActions.getNewMetadata()), updatedActions.getNewProtocol() == updatedActions.getOldProtocol() ?