[WIP/PoC] Centralize KFP driver as standalone ml-pipeline-driver service

.github/actions/deploy/action.yml

@@ -2,10 +2,6 @@ name: "Set up KFP on KinD"
 description: "Step to start and configure KFP on Kind"
 
 inputs:
-  cluster_name:
-    description: "Kind cluster name"
-    default: 'kfp'
-    required: false
   pipeline_store:
     description: "Flag to deploy KFP with K8s Native API"
     default: 'database'
@@ -74,12 +70,6 @@ runs:
           docker image rm ${{ inputs.image_registry }}/$app:${{ inputs.image_tag }}
           docker image rm localhost:5000/$app:${{ inputs.image_tag }}
         done
-        docker pull python:3.11
-        docker pull registry.access.redhat.com/ubi9/python-311:latest
-        kind load docker-image python:3.11 --name ${{ inputs.cluster_name }}
-        kind load docker-image registry.access.redhat.com/ubi9/python-311:latest --name ${{ inputs.cluster_name }}
-        docker image rm python:3.11
-        docker image rm registry.access.redhat.com/ubi9/python-311:latest
 
     - name: Configure Args
       shell: bash

.github/resources/manifests/base/apiserver-env.yaml

@@ -21,8 +21,6 @@ spec:
       containers:
         - name: ml-pipeline-api-server
           env:
-            - name: V2_DRIVER_IMAGE
-              value: kind-registry:5000/driver
             - name: V2_LAUNCHER_IMAGE
               value: kind-registry:5000/launcher
             - name: LOG_LEVEL

.github/resources/manifests/kubernetes-native/default/kustomization.yaml

@@ -36,6 +36,9 @@ images:
   - name: ghcr.io/kubeflow/kfp-metadata-envoy
     newName: kind-registry:5000/metadata-envoy
     newTag: latest
+  - name: ghcr.io/kubeflow/kfp-driver
+    newName: kind-registry:5000/driver
+    newTag: latest
 
 patches:
   - path: ../../base/apiserver-env.yaml

.github/resources/manifests/multiuser/artifact-proxy/kustomization.yaml

@@ -36,6 +36,9 @@ images:
   - name: ghcr.io/kubeflow/kfp-metadata-envoy
     newName: kind-registry:5000/metadata-envoy
     newTag: latest
+  - name: ghcr.io/kubeflow/kfp-driver
+    newName: kind-registry:5000/driver
+    newTag: latest
 
 patches:
   - path: ../pipeline-crd-rbac.yaml
@@ -96,4 +99,3 @@ replacements:
           name: cache-server
         fieldPaths:
           - spec.template.spec.dnsConfig.searches.[=NAMESPACE.svc.cluster.local]
-

.github/resources/manifests/multiuser/cache-disabled/kustomization.yaml

@@ -36,6 +36,9 @@ images:
   - name: ghcr.io/kubeflow/kfp-metadata-envoy
     newName: kind-registry:5000/metadata-envoy
     newTag: latest
+  - name: ghcr.io/kubeflow/kfp-driver
+    newName: kind-registry:5000/driver
+    newTag: latest
 
 patches:
   - path: ../pipeline-crd-rbac.yaml

.github/resources/manifests/multiuser/default/kustomization.yaml

@@ -36,6 +36,9 @@ images:
   - name: ghcr.io/kubeflow/kfp-metadata-envoy
     newName: kind-registry:5000/metadata-envoy
     newTag: latest
+  - name: ghcr.io/kubeflow/kfp-driver
+    newName: kind-registry:5000/driver
+    newTag: latest
 
 patches:
   - path: ../pipeline-crd-rbac.yaml

.github/resources/manifests/standalone/cache-disabled/kustomization.yaml

@@ -36,6 +36,9 @@ images:
   - name: ghcr.io/kubeflow/kfp-metadata-envoy
     newName: kind-registry:5000/metadata-envoy
     newTag: latest
+  - name: ghcr.io/kubeflow/kfp-driver
+    newName: kind-registry:5000/driver
+    newTag: latest
 
 patches:
   - path: ../../base/apiserver-env.yaml

.github/resources/manifests/standalone/default/kustomization.yaml

@@ -36,6 +36,9 @@ images:
   - name: ghcr.io/kubeflow/kfp-metadata-envoy
     newName: kind-registry:5000/metadata-envoy
     newTag: latest
+  - name: ghcr.io/kubeflow/kfp-driver
+    newName: kind-registry:5000/driver
+    newTag: latest
 
 patches:
   - path: ../../base/apiserver-env.yaml

.github/resources/manifests/standalone/proxy/kustomization.yaml

@@ -36,6 +36,9 @@ images:
   - name: ghcr.io/kubeflow/kfp-metadata-envoy
     newName: kind-registry:5000/metadata-envoy
     newTag: latest
+  - name: ghcr.io/kubeflow/kfp-driver
+    newName: kind-registry:5000/driver
+    newTag: latest
 
 patches:
   - path: ../../base/apiserver-env.yaml

.github/resources/manifests/standalone/tls-enabled/kustomization.yaml

@@ -36,6 +36,9 @@ images:
   - name: ghcr.io/kubeflow/kfp-metadata-envoy
     newName: kind-registry:5000/metadata-envoy
     newTag: latest
+  - name: ghcr.io/kubeflow/kfp-driver
+    newName: kind-registry:5000/driver
+    newTag: latest
 
 patches:
   - path: ../../base/apiserver-env.yaml

.github/resources/scripts/collect-logs.sh

@@ -29,6 +29,15 @@ function check_namespace {
     return 0
 }
 
+function describe_argo_workflows {
+    local NAMESPACE=$1
+    echo "===== Argo Workflows list ====="
+    kubectl describe wf  -n "${NAMESPACE}"
+    echo "===== Argo Workflows data ====="
+    kubectl get events -n "${NAMESPACE}" --field-selector involvedObject.kind=Workflow --sort-by='.metadata.creationTimestamp'
+    echo "==============================="
+}
+
 function display_pod_info {
     local NAMESPACE=$1
 
@@ -52,7 +61,13 @@ function display_pod_info {
             kubectl describe pod "${POD_NAME}" -n "${NAMESPACE}" | grep -A 100 Events || echo "No events found for pod ${POD_NAME}."
 
             echo "----- LOGS -----"
-            kubectl logs "${POD_NAME}" -n "${NAMESPACE}" || echo "No logs found for pod ${POD_NAME}."
+            if [[ "${POD_NAME}" == *-agent* ]]; then
+                kubectl logs "${POD_NAME}" -n "${NAMESPACE}" -c driver-plugin || \
+                    echo "No logs found for pod ${POD_NAME}."
+            else
+                kubectl logs "${POD_NAME}" -n "${NAMESPACE}" || \
+                    echo "No logs found for pod ${POD_NAME}."
+            fi
 
             echo "==========================="
             echo ""
@@ -64,6 +79,7 @@ function display_pod_info {
 
 if check_namespace "$NS"; then
     display_pod_info "$NS"
+    describe_argo_workflows "$NS"
 else
     exit 0
 fi

.github/resources/scripts/kfp-readiness/wait_for_pods.py

@@ -30,7 +30,8 @@ def get_pod_statuses():
     statuses = {}
     for pod in pods.items:
         pod_name = pod.metadata.name
-        if "system" not in pod_name:
+        # This filter is safe: 'ml-pipeline-persistenceagent-<guid>' will not be excluded and will be processed.
+        if not ("system" in pod_name or pod_name.endswith("-agent")):
             pod_status = pod.status.phase
             container_statuses = pod.status.container_statuses or []
             ready = 0

.github/workflows/api-server-tests.yml

@@ -66,13 +66,13 @@ jobs:
         k8s_version: [ "v1.34.0" ]
         cache_enabled: [ "true", "false" ]
         proxy: [ "true", "false" ]
-        argo_version: [ "v3.7.3", "v3.5.14" ]
+        argo_version: [ "v3.7.3", "v3.6.7" ]
         pipeline_store: [ "database" ]
         pod_to_pod_tls_enabled: [ "false" ]
         include:
           - k8s_version: "v1.29.2"
             cache_enabled: "true"
-            argo_version: "v3.7.3"
+            argo_version: "v3.6.7"
           - k8s_version: "v1.29.2"
             cache_enabled: "true"
             argo_version: "v3.5.14"
@@ -111,7 +111,7 @@ jobs:
         shell: bash
         if: ${{ matrix.pod_to_pod_tls_enabled == 'true'}}
         run: |
-          kubectl get secret kfp-api-tls-cert -n kubeflow -o jsonpath='{.data.ca\.crt}' | base64 -d > "${{ github.workspace }}/ca.crt"
+          kubectl get secret argo-workflows-agent-ca-certificates -n kubeflow -o jsonpath='{.data.ca\.crt}' | base64 -d > "${{ github.workspace }}/ca.crt"
           echo "CA_CERT_PATH=${{ github.workspace }}/ca.crt" >> "$GITHUB_ENV"
 
 
@@ -167,7 +167,7 @@ jobs:
         k8s_version: [ "v1.34.0", "v1.29.2" ]
         cache_enabled: [ "true" ]
         uploadPipelinesWithKubernetesClient: [ "true", "false" ]
-        argo_version: [ "v3.7.3", "v3.5.14" ]
+        argo_version: [ "v3.7.3", "v3.6.7" ]
         pipeline_store: [ "kubernetes" ]
         pod_to_pod_tls_enabled: [ "false" ]
         include:

.github/workflows/e2e-test.yml

@@ -63,9 +63,9 @@ jobs:
       matrix:
         k8s_version: ["v1.34.0"]
         cache_enabled: ["true", "false"]
-        argo_version: [ "v3.7.3", "v3.5.14"]
+        argo_version: [ "v3.7.3", "v3.6.7", "v3.5.14"]
         proxy: [ "false" ]
-        test_label: [ "E2ECritical", "E2EEssential", "E2EParallelNested" ]
+        test_label: [ "E2ECritical" ]
         pod_to_pod_tls_enabled: [ "false" ]
         include:
           - k8s_version: "v1.29.2"
@@ -76,6 +76,9 @@ jobs:
             cache_enabled: "false"
             proxy: "true"
             test_label: "E2EProxy"
+          - k8s_version: "v1.34.0"
+            cache_enabled: "false"
+            test_label: "E2EEssential"
           - k8s_version: "v1.34.0"
             cache_enabled: "false"
             test_label: "E2EFailure"
@@ -112,7 +115,7 @@ jobs:
         shell: bash
         if: ${{ matrix.pod_to_pod_tls_enabled == 'true'}}
         run: |
-          kubectl get secret kfp-api-tls-cert -n kubeflow -o jsonpath='{.data.ca\.crt}' | base64 -d > "${{ github.workspace }}/ca.crt"
+          kubectl get secret argo-workflows-agent-ca-certificates -n kubeflow -o jsonpath='{.data.ca\.crt}' | base64 -d > "${{ github.workspace }}/ca.crt"
           echo "CA_CERT_PATH=${{ github.workspace }}/ca.crt" >> "$GITHUB_ENV"
       - name: Configure Input Variables
         shell: bash
@@ -128,10 +131,6 @@ jobs:
             NAMESPACE=${{ inputs.namespace }}
           fi
 
-          if [ "$TEST_LABEL" == "E2EParallelNested" ]; then
-            NUMBER_OF_NODES=3
-          fi
-
           {
             echo "NUMBER_OF_NODES=$NUMBER_OF_NODES"
             echo "TEST_LABEL=$TEST_LABEL"

.github/workflows/image-builds-master.yml

@@ -30,9 +30,6 @@ jobs:
           - name: kfp-launcher
             dockerfile: backend/Dockerfile.launcher
             context: .
-          - name: kfp-driver
-            dockerfile: backend/Dockerfile.driver
-            context: .
           - name: kfp-cache-deployer
             dockerfile: backend/src/cache/deployer/Dockerfile
             context: .
@@ -48,6 +45,9 @@ jobs:
           - name: kfp-inverse-proxy-agent
             dockerfile: proxy/Dockerfile
             context: ./proxy
+          - name: kfp-driver
+            dockerfile: backend/Dockerfile.driver
+            context: .
 
         arch:
           - arch_name: amd64
@@ -93,12 +93,12 @@ jobs:
           - image: kfp-viewer-crd-controller
           - image: kfp-visualization-server
           - image: kfp-launcher
-          - image: kfp-driver
           - image: kfp-cache-deployer
           - image: kfp-cache-server
           - image: kfp-metadata-writer
           - image: kfp-metadata-envoy
           - image: kfp-inverse-proxy-agent
+          - image: kfp-driver
     uses: ./.github/workflows/create-manifest.yml
     secrets: inherit
     with:

.github/workflows/image-builds-release.yml

@@ -59,9 +59,6 @@ jobs:
           - name: kfp-launcher
             dockerfile: backend/Dockerfile.launcher
             context: .
-          - name: kfp-driver
-            dockerfile: backend/Dockerfile.driver
-            context: .
           - name: kfp-cache-deployer
             dockerfile: backend/src/cache/deployer/Dockerfile
             context: .
@@ -77,6 +74,9 @@ jobs:
           - name: kfp-inverse-proxy-agent
             dockerfile: proxy/Dockerfile
             context: ./proxy
+          - name: kfp-driver
+            dockerfile: backend/Dockerfile.driver
+            context: .
 
         arch:
           - arch_name: amd64
@@ -123,7 +123,6 @@ jobs:
           - image: kfp-viewer-crd-controller
           - image: kfp-visualization-server
           - image: kfp-launcher
-          - image: kfp-driver
           - image: kfp-cache-deployer
           - image: kfp-cache-server
           - image: kfp-metadata-writer

.github/workflows/image-builds.yml

@@ -41,9 +41,6 @@ jobs:
           - image: launcher
             dockerfile: backend/Dockerfile.launcher
             context: .
-          - image: driver
-            dockerfile: backend/Dockerfile.driver
-            context: .
           - image: frontend
             dockerfile: frontend/Dockerfile
             context: .
@@ -62,6 +59,9 @@ jobs:
           - image: cache-server
             dockerfile: backend/Dockerfile.cacheserver
             context: .
+          - image: driver
+            dockerfile: backend/Dockerfile.driver
+            context: .
           - image: metadata-envoy
             dockerfile: third_party/metadata_envoy/Dockerfile
             context: .

.github/workflows/kfp-sdk-tests.yml

@@ -20,7 +20,7 @@ concurrency:
 jobs:
   sdk-tests:
     runs-on: ubuntu-latest
-    timeout-minutes: 50
+    timeout-minutes: 45
     strategy:
       matrix:
         python-version: ['3.9', '3.13']
@@ -49,7 +49,6 @@ jobs:
         pip install pytest
         pip install docker
         pip install pytest-cov
-        pip install pytest-xdist
         pip install google_cloud_pipeline_components
         pip install requests==2.28.1
 
@@ -59,6 +58,5 @@ jobs:
         SETUP_ENV: false
         REPO_NAME: ${{ github.repository }}
         PULL_NUMBER: ${{ github.event.pull_request.number }}
-        PYTEST_PARALLEL_WORKERS: 2
       run: |
         ./test/presubmit-tests-sdk.sh

.github/workflows/legacy-v2-api-integration-tests.yml

@@ -79,7 +79,7 @@ jobs:
         shell: bash
         if: ${{ matrix.pod_to_pod_tls_enabled == 'true' }}
         run: |
-          kubectl get secret kfp-api-tls-cert -n kubeflow -o jsonpath='{.data.ca\.crt}' | base64 -d > "${{ github.workspace }}/ca.crt"
+          kubectl get secret argo-workflows-agent-ca-certificates -n kubeflow -o jsonpath='{.data.ca\.crt}' | base64 -d > "${{ github.workspace }}/ca.crt"
           echo "CA_CERT_PATH=${{ github.workspace }}/ca.crt" >> "$GITHUB_ENV"
 
       - name: Forward MLMD port

.github/workflows/sdk-component-yaml.yml

@@ -39,10 +39,13 @@ jobs:
         uses: ./.github/actions/kfp-k8s
 
       - name: Install Test dependencies
-        run: make test-component-yaml-install-deps
+        run: |
+          python3 -m pip install pytest
+          python3 -m pip install pytest-asyncio-cooperative==0.37.0
+          python3 -m pip install --upgrade pip
 
       - name: Run component YAML tests
         env:
           # We setup the env in the CI
           SETUP_ENV: false
-        run: make test-component-yaml-run
+        run: ./test/presubmit-component-yaml.sh

Makefile

@@ -20,13 +20,3 @@ ginkgo:
 	mkdir -p $(BIN_DIR)
 	GOBIN=$(BIN_DIR) go install github.com/onsi/ginkgo/v2/ginkgo@latest
 	@echo "Ginkgo installed to $(BIN_DIR)/ginkgo"
-
-# Component YAML Tests
-.PHONY: test-component-yaml-install-deps
-test-component-yaml-install-deps:
-	python3 -m pip install pytest
-	python3 -m pip install pytest-asyncio-cooperative==0.37.0
-
-.PHONY: test-component-yaml-run
-test-component-yaml-run:
-	./test/presubmit-component-yaml.sh