Test yaml

.github/workflows/java-pr.yml

@@ -57,112 +57,23 @@ env:
 permissions: write-all
 
 jobs:
-  spotless_check:
-    name: Spotless
-    timeout-minutes: 10
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
-      - name: Setup Environment
-        id: setup-env
-        uses: ./.github/actions/setup-env
-      - name: Run Spotless
-        run: ./cicd/run-spotless
-  checkstyle_check:
-    name: Checkstyle
-    timeout-minutes: 10
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
-      - name: Setup Environment
-        id: setup-env
-        uses: ./.github/actions/setup-env
-      - name: Run Checkstyle
-        run: ./cicd/run-checkstyle
   java_build:
     name: Build
     timeout-minutes: 60
     runs-on: [self-hosted, it]
     steps:
       - name: Checkout Code
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+        uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
       - name: Setup Environment
         id: setup-env
         uses: ./.github/actions/setup-env
       - name: Run Build
         run: ./cicd/run-build
       - name: Cleanup Java Environment
         uses: ./.github/actions/cleanup-java-env
-  java_unit_tests:
-    name: Unit Tests
-    needs: [java_build]
-    timeout-minutes: 60
-    runs-on: [self-hosted, it]
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
-      - name: Setup Environment
-        id: setup-env
-        uses: ./.github/actions/setup-env
-      - name: Run Unit Tests
-        run: ./cicd/run-unit-tests
-      - name: Upload Unit Tests Report
-        uses: actions/upload-artifact@v5
-        if: always() # always run even if the previous step fails
-        with:
-          name: surefire-unit-test-results
-          path: |
-            **/surefire-reports/TEST-*.xml
-            **/surefire-reports/*.html
-            **/surefire-reports/html/**
-          retention-days: 1
-      - name: Upload coverage reports to Codecov
-        uses: codecov/codecov-action@v5.5.1
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          slug: GoogleCloudPlatform/DataflowTemplates
-          files: 'target/site/jacoco-aggregate/jacoco.xml'
-      - name: Cleanup Java Environment
-        uses: ./.github/actions/cleanup-java-env
-        if: always()
-  java_integration_smoke_tests_templates:
-    name: Dataflow Templates Integration Smoke Tests
-    needs: [spotless_check, checkstyle_check, java_build, java_unit_tests]
-    timeout-minutes: 60
-    # Run on any runner that matches all the specified runs-on values.
-    runs-on: [self-hosted, it]
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
-      - name: Setup Environment
-        id: setup-env
-        uses: ./.github/actions/setup-env
-      - name: Run Integration Smoke Tests
-        run: |
-          ./cicd/run-it-smoke-tests \
-          --modules-to-build="DEFAULT" \
-          --it-region="${{ env.IT_REGION }}" \
-          --it-project="cloud-teleport-testing" \
-          --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \
-          --it-private-connectivity="datastream-connect-2"
-      - name: Upload Smoke Tests Report
-        uses: actions/upload-artifact@v5
-        if: always() # always run even if the previous step fails
-        with:
-          name: surefire-smoke-test-results
-          path: |
-            **/surefire-reports/TEST-*.xml
-            **/surefire-reports/*.html
-            **/surefire-reports/html/**
-          retention-days: 1
-      - name: Cleanup Java Environment
-        uses: ./.github/actions/cleanup-java-env
-        if: always()
   java_integration_tests_templates:
     name: Dataflow Templates Integration Tests
-    needs: [java_integration_smoke_tests_templates]
+    needs: [java_build]
     timeout-minutes: 240
     # Run on any runner that matches all the specified runs-on values.
     runs-on: [self-hosted, it]
@@ -175,7 +86,8 @@ jobs:
       - name: Run Integration Tests
         run: |
           ./cicd/run-it-tests \
-          --modules-to-build="DEFAULT" \
+          --modules-to-build="YAML" \
+          --modules-to-build="YAML" \
           --it-region="${{ env.IT_REGION }}" \
           --it-project="cloud-teleport-testing" \
           --it-integration-test-parallelism=6 \
@@ -207,26 +119,6 @@ jobs:
       - name: Cleanup Java Environment
         uses: ./.github/actions/cleanup-java-env
         if: always()
-  java_load_tests_templates:
-    if: contains(github.event.pull_request.labels.*.name, 'run-load-tests')
-    name: Dataflow Templates Load Tests
-    needs: [spotless_check, checkstyle_check, java_build, java_unit_tests, java_integration_tests_templates]
-    timeout-minutes: 600
-    # Run on any runner that matches all the specified runs-on values.
-    runs-on: [self-hosted, perf]
-    steps:
-    - name: Checkout Code
-      uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
-    - name: Setup Environment
-      id: setup-env
-      uses: ./.github/actions/setup-env
-    - name: Run Load Tests
-      run: |
-        ./cicd/run-load-tests \
-        --it-region="${{ env.IT_REGION }}" \
-        --it-project="cloud-teleport-testing" \
-        --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \
-        --it-private-connectivity="datastream-connect-2"
-    - name: Cleanup Java Environment
-      uses: ./.github/actions/cleanup-java-env
-      if: always()
+      - name: Cleanup Java Environment
+        uses: ./.github/actions/cleanup-java-env
+        if: always()

cicd/internal/flags/common-flags.go

@@ -28,6 +28,7 @@ const (
 	SPANNER    = "SPANNER"
 	BIGTABLE   = "BIGTABLE"
 	DATASTREAM = "DATASTREAM"
+	YAML       = "YAML"
 )
 
 // Avoid making these vars public.
@@ -64,6 +65,7 @@ var (
 			"v2/datastream-to-mongodb/",
 			"v2/datastream-to-sql/",
 		},
+		YAML: {"yaml/"},
 	}
 )
 

it/iceberg/pom.xml

@@ -27,6 +27,7 @@
 
     <properties>
         <iceberg.version>1.6.1</iceberg.version>
+        <bigdataoss.version>3.1.10</bigdataoss.version>
     </properties>
 
     <dependencies>
@@ -69,6 +70,26 @@
             <artifactId>parquet-hadoop</artifactId>
             <version>1.13.1</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.iceberg</groupId>
+            <artifactId>iceberg-gcp</artifactId>
+            <version>${iceberg.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.cloud.bigdataoss</groupId>
+            <artifactId>gcsio</artifactId>
+            <version>${bigdataoss.version}</version>
+       </dependency>
+        <dependency>
+            <groupId>com.google.cloud.bigdataoss</groupId>
+            <artifactId>gcs-connector</artifactId>
+            <version>${bigdataoss.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.cloud.bigdataoss</groupId>
+            <artifactId>util-hadoop</artifactId>
+            <version>${bigdataoss.version}</version>
+        </dependency>
     </dependencies>
 
 

yaml/pom.xml

@@ -36,6 +36,7 @@
   <properties>
     <maven.compiler.source>17</maven.compiler.source>
     <maven.compiler.target>17</maven.compiler.target>
+    <postgresql.version>42.6.1</postgresql.version>
   </properties>
 
   <dependencies>
@@ -65,6 +66,24 @@
       <artifactId>beam-it-kafka</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>com.google.cloud.teleport</groupId>
+      <artifactId>it-jdbc</artifactId>
+      <version>${project.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.google.cloud.teleport</groupId>
+      <artifactId>iceberg</artifactId>
+      <version>${project.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.postgresql</groupId>
+      <artifactId>postgresql</artifactId>
+      <version>${postgresql.version}</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>