NOSNOW: Revert Spark4 work for the upcoming 3.1.9 release

.github/scripts/decrypt_secret.sh

@@ -7,11 +7,6 @@
 echo "Usage: decrypt_secret.sh output_file_name decrypted_file_name"
 echo "       Note: environment variable SNOWFLAKE_TEST_CONFIG_SECRET should be set for descryption."
 
-if [ -z "$SNOWFLAKE_TEST_CONFIG_SECRET" ]; then
-  echo "WARNING: SNOWFLAKE_TEST_CONFIG_SECRET is not set (expected for forks and external PRs)."
-  echo "Skipping decryption — integration tests will not run."
-  exit 0
-fi
-
 # --batch to prevent interactive command --yes to assume "yes" for questions
 gpg --quiet --batch --yes --decrypt --passphrase="$SNOWFLAKE_TEST_CONFIG_SECRET" --output $1 $2
+

.github/scripts/run-tests-github.sh

@@ -26,30 +26,10 @@ if [ "$SNOWFLAKE_TEST_ACCOUNT" == "aws" -a "$SPARK_CONN_ENV_USE_COPY_UNLOAD" ==
   export EXTRA_TEST_FOR_COVERAGE=true
 fi
 
-# If credentials file is missing (forks, external PRs), run unit tests only
-if [ "$INTEGRATION_TESTS" == "true" ] && [ ! -f "snowflake.travis.json" ]; then
-  echo ""
-  echo "========================================================================"
-  echo "WARNING: Snowflake credentials not available (snowflake.travis.json"
-  echo "         not found). This is expected for forks and external PRs."
-  echo ""
-  echo "         Integration tests are SKIPPED — do NOT treat this as a full"
-  echo "         green build."
-  echo "========================================================================"
-  echo ""
-
-  # Signal to the workflow that integration tests were skipped
-  if [ -n "$GITHUB_OUTPUT" ]; then
-    echo "INTEGRATION_TESTS_SKIPPED=true" >> "$GITHUB_OUTPUT"
-  fi
-
-  unset INTEGRATION_TESTS
-fi
-
 if [ "$INTEGRATION_TESTS" != "true" ]; then
   # Run only test
-  sbt -DsparkVersion=$SPARK_VERSION -DscalaVersion=$SPARK_SCALA_VERSION clean evicted coverage test coverageReport
+  sbt -Dspark.testVersion=$SPARK_VERSION ++$SPARK_SCALA_VERSION clean coverage test coverageReport
 else
   # Run both test and it
-  sbt -DsparkVersion=$SPARK_VERSION -DscalaVersion=$SPARK_SCALA_VERSION clean evicted coverage test it:test coverageReport
+  sbt -Dspark.version=$SPARK_VERSION ++$SPARK_SCALA_VERSION clean coverage test it:test coverageReport
 fi

.github/workflows/IntegrationTest_2.12.yml

@@ -12,28 +12,26 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
      matrix:
-       scala_version: [ '2.12.18' ]
-       spark_version: [ '3.5.8' ]
+       scala_version: [ '2.12.11' ]
+       spark_version: [ '3.5.0' ]
        use_copy_unload: [ 'true', 'false' ]
        cloud_provider: [ 'aws', 'azure' ]
        # run_query_in_async can be removed after async mode is stable
        run_query_in_async: [ 'true', 'false' ]
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v2
     - name: Set up JDK 1.8
-      uses: actions/setup-java@v4
+      uses: actions/setup-java@v1
       with:
-        distribution: 'temurin'
-        java-version: '8'
+        java-version: 1.8
 
     - name: Decrypt snowflake.json for testing
       run: ./.github/scripts/decrypt_secret.sh snowflake.travis.json snowflake.travis.json.gpg
       env:
         SNOWFLAKE_TEST_CONFIG_SECRET: ${{ secrets.SNOWFLAKE_TEST_CONFIG_SECRET }}
 
     - name: Run tests
-      id: run-tests
       run: ./.github/scripts/run-tests-github.sh
       env:
         INTEGRATION_TESTS: true
@@ -43,11 +41,7 @@ jobs:
         SPARK_CONN_ENV_USE_COPY_UNLOAD: ${{ matrix.use_copy_unload }}
         SPARK_CONN_ENV_INTERNAL_EXECUTE_QUERY_IN_SYNC_MODE: ${{ matrix.run_query_in_async }}
 
-    - name: Warn if integration tests were skipped
-      if: steps.run-tests.outputs.INTEGRATION_TESTS_SKIPPED == 'true'
-      run: echo "::warning::Integration tests were SKIPPED — Snowflake credentials not available (fork or external PR). Unit tests passed but this is NOT a full green build."
-
-    - uses: codecov/codecov-action@v4
+    - uses: codecov/codecov-action@v2
       if: matrix.use_copy_unload != 'true' || matrix.cloud_provider != 'gcp'
       with:
         fail_ci_if_error: false

.github/workflows/IntegrationTest_2.13.yml

@@ -12,43 +12,26 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
      matrix:
-       spark_version: [ '3.5.8', '4.0.0', '4.1.0' ]
+       scala_version: [ '2.13.9' ]
+       spark_version: [ '3.5.0' ]
        use_copy_unload: [ 'true', 'false' ]
        cloud_provider: [ 'aws', 'azure' ]
+       # run_query_in_async can be removed after async mode is stable
        run_query_in_async: [ 'true', 'false' ]
-       include:
-         # Default Scala/Java for all Spark versions
-         - spark_version: '3.5.8'
-           scala_version: '2.13.16'
-           java_version: '8'
-         - spark_version: '4.0.0'
-           scala_version: '2.13.16'
-           java_version: '17'
-         - spark_version: '4.1.0'
-           scala_version: '2.13.17'
-           java_version: '17'
-       exclude:
-         # Spark 4.1 keeps exclusions until 4.0 is validated in CI
-         - spark_version: '4.1.0'
-           use_copy_unload: 'true'
-         - spark_version: '4.1.0'
-           run_query_in_async: 'true'
 
     steps:
-    - uses: actions/checkout@v4
-    - name: Set up JDK
-      uses: actions/setup-java@v4
+    - uses: actions/checkout@v2
+    - name: Set up JDK 1.8
+      uses: actions/setup-java@v1
       with:
-        distribution: 'temurin'
-        java-version: ${{ matrix.java_version }}
+        java-version: 1.8
 
     - name: Decrypt snowflake.json for testing
       run: ./.github/scripts/decrypt_secret.sh snowflake.travis.json snowflake.travis.json.gpg
       env:
         SNOWFLAKE_TEST_CONFIG_SECRET: ${{ secrets.SNOWFLAKE_TEST_CONFIG_SECRET }}
 
     - name: Run tests
-      id: run-tests
       run: ./.github/scripts/run-tests-github.sh
       env:
         INTEGRATION_TESTS: true
@@ -58,11 +41,7 @@ jobs:
         SPARK_CONN_ENV_USE_COPY_UNLOAD: ${{ matrix.use_copy_unload }}
         SPARK_CONN_ENV_INTERNAL_EXECUTE_QUERY_IN_SYNC_MODE: ${{ matrix.run_query_in_async }}
 
-    - name: Warn if integration tests were skipped
-      if: steps.run-tests.outputs.INTEGRATION_TESTS_SKIPPED == 'true'
-      run: echo "::warning::Integration tests were SKIPPED — Snowflake credentials not available (fork or external PR). Unit tests passed but this is NOT a full green build."
-
-    - uses: codecov/codecov-action@v4
+    - uses: codecov/codecov-action@v2
       if: matrix.use_copy_unload != 'true' || matrix.cloud_provider != 'gcp'
       with:
         fail_ci_if_error: false

.github/workflows/IntegrationTest_gcp_2.12.yml

@@ -12,28 +12,26 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
      matrix:
-       scala_version: [ '2.12.18' ]
-       spark_version: [ '3.5.8' ]
+       scala_version: [ '2.12.11' ]
+       spark_version: [ '3.5.0' ]
        use_copy_unload: [ 'false' ]
        cloud_provider: [ 'gcp' ]
        # run_query_in_async can be removed after async mode is stable
        run_query_in_async: [ 'true', 'false' ]
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v2
     - name: Set up JDK 1.8
-      uses: actions/setup-java@v4
+      uses: actions/setup-java@v1
       with:
-        distribution: 'temurin'
-        java-version: '8'
+        java-version: 1.8
 
     - name: Decrypt snowflake.json for testing
       run: ./.github/scripts/decrypt_secret.sh snowflake.travis.json snowflake.travis.json.gpg
       env:
         SNOWFLAKE_TEST_CONFIG_SECRET: ${{ secrets.SNOWFLAKE_TEST_CONFIG_SECRET }}
 
     - name: Run tests
-      id: run-tests
       run: ./.github/scripts/run-tests-github.sh
       env:
         INTEGRATION_TESTS: true
@@ -43,11 +41,7 @@ jobs:
         SPARK_CONN_ENV_USE_COPY_UNLOAD: ${{ matrix.use_copy_unload }}
         SPARK_CONN_ENV_INTERNAL_EXECUTE_QUERY_IN_SYNC_MODE: ${{ matrix.run_query_in_async }}
 
-    - name: Warn if integration tests were skipped
-      if: steps.run-tests.outputs.INTEGRATION_TESTS_SKIPPED == 'true'
-      run: echo "::warning::Integration tests were SKIPPED — Snowflake credentials not available (fork or external PR). Unit tests passed but this is NOT a full green build."
-
-    - uses: codecov/codecov-action@v4
+    - uses: codecov/codecov-action@v2
       if: matrix.use_copy_unload != 'true' || matrix.cloud_provider != 'gcp'
       with:
         fail_ci_if_error: false

.github/workflows/IntegrationTest_gcp_2.13.yml

@@ -12,39 +12,26 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
      matrix:
-       spark_version: [ '3.5.8', '4.0.0', '4.1.0' ]
+       scala_version: [ '2.13.9' ]
+       spark_version: [ '3.5.0' ]
        use_copy_unload: [ 'false' ]
        cloud_provider: [ 'gcp' ]
+       # run_query_in_async can be removed after async mode is stable
        run_query_in_async: [ 'true', 'false' ]
-       include:
-         - spark_version: '3.5.8'
-           scala_version: '2.13.16'
-           java_version: '8'
-         - spark_version: '4.0.0'
-           scala_version: '2.13.16'
-           java_version: '17'
-         - spark_version: '4.1.0'
-           scala_version: '2.13.17'
-           java_version: '17'
-       exclude:
-         - spark_version: '4.1.0'
-           run_query_in_async: 'true'
 
     steps:
-    - uses: actions/checkout@v4
-    - name: Set up JDK
-      uses: actions/setup-java@v4
+    - uses: actions/checkout@v2
+    - name: Set up JDK 1.8
+      uses: actions/setup-java@v1
       with:
-        distribution: 'temurin'
-        java-version: ${{ matrix.java_version }}
+        java-version: 1.8
 
     - name: Decrypt snowflake.json for testing
       run: ./.github/scripts/decrypt_secret.sh snowflake.travis.json snowflake.travis.json.gpg
       env:
         SNOWFLAKE_TEST_CONFIG_SECRET: ${{ secrets.SNOWFLAKE_TEST_CONFIG_SECRET }}
 
     - name: Run tests
-      id: run-tests
       run: ./.github/scripts/run-tests-github.sh
       env:
         INTEGRATION_TESTS: true
@@ -54,11 +41,7 @@ jobs:
         SPARK_CONN_ENV_USE_COPY_UNLOAD: ${{ matrix.use_copy_unload }}
         SPARK_CONN_ENV_INTERNAL_EXECUTE_QUERY_IN_SYNC_MODE: ${{ matrix.run_query_in_async }}
 
-    - name: Warn if integration tests were skipped
-      if: steps.run-tests.outputs.INTEGRATION_TESTS_SKIPPED == 'true'
-      run: echo "::warning::Integration tests were SKIPPED — Snowflake credentials not available (fork or external PR). Unit tests passed but this is NOT a full green build."
-
-    - uses: codecov/codecov-action@v4
+    - uses: codecov/codecov-action@v2
       if: matrix.use_copy_unload != 'true' || matrix.cloud_provider != 'gcp'
       with:
         fail_ci_if_error: false

README.md

@@ -5,11 +5,9 @@
 [![codecov](https://codecov.io/gh/snowflakedb/spark-snowflake/branch/master/graph/badge.svg)](https://codecov.io/gh/snowflakedb/spark-snowflake)
 [![License](http://img.shields.io/:license-Apache%202-brightgreen.svg)](http://www.apache.org/licenses/LICENSE-2.0.txt)
 
-The main version of `spark-snowflake` works with Spark 3.5 and 4.0.
+The main version of `spark-snowflake` works with Spark 2.4. For use with Spark 2.3 and 2.2, please use tag `vx.x.x-spark_2.3` and `vx.x.x-spark_2.2`. 
 
-For earlier versions of Spark (e.g. 2.4, 2.3 and 2.2), please use tag `vx.x.x-spark_[version]`.
-
-To use it, provide the dependency for Spark in the form of `net.snowflake:spark-snowflake_$SCALA_VERSION:$RELEASE-spark_$SPARKVERSION`, e.g. `net.snowflake:spark-snowflake_2.11:2.2.2-spark_3.5`. See [Maven Central](http://search.maven.org/#search%7Cga%7C1%7Cspark-snowflake) for more info.
+To use it, provide the dependency for Spark in the form of `net.snowflake:spark-snowflake_$SCALA_VERSION:$RELEASE`, e.g. `net.snowflake:spark-snowflake_2.11:2.2.2`. See [Maven Central](http://search.maven.org/#search%7Cga%7C1%7Cspark-snowflake) for more info.
 
 For a version working with Spark 1.5 and 1.6, please use `branch-1.x`. Artifacts of that version are also available in the Snowflake UI.