diff --git a/.github/resources/manifests/base/driver-plugin-cm-path.yaml b/.github/resources/manifests/base/driver-plugin-cm-path.yaml new file mode 100644 index 00000000000..9a8cf31eece --- /dev/null +++ b/.github/resources/manifests/base/driver-plugin-cm-path.yaml @@ -0,0 +1,42 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ml-pipeline-driver-agent +data: + sidecar.container: | + name: driver-plugin + image: kind-registry:5000/driver:ci + imagePullPolicy: IfNotPresent + env: + - name: LOG_ACCESS_KEY + valueFrom: + secretKeyRef: + name: mlpipeline-minio-artifact + key: accesskey + - name: LOG_SECRET_KEY + valueFrom: + secretKeyRef: + name: mlpipeline-minio-artifact + key: secretkey + ports: + - containerPort: 8080 + resources: + requests: + cpu: "0.1" + memory: "64Mi" + limits: + cpu: "0.5" + memory: "0.5Gi" + securityContext: + runAsNonRoot: true + runAsUser: 65534 + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: var-run-argo + mountPath: /kfp/log + readOnly: false diff --git a/.github/resources/manifests/kubernetes-native/default/kustomization.yaml b/.github/resources/manifests/kubernetes-native/default/kustomization.yaml index 8520b973639..0ddcc2683de 100644 --- a/.github/resources/manifests/kubernetes-native/default/kustomization.yaml +++ b/.github/resources/manifests/kubernetes-native/default/kustomization.yaml @@ -42,6 +42,10 @@ patches: target: kind: Deployment name: ml-pipeline + - path: ../../base/driver-plugin-cm-path.yaml + target: + kind: ConfigMap + name: ml-pipeline-driver-agent - path: ../../base/grpc-specs.yaml target: kind: Deployment diff --git a/.github/resources/manifests/multiuser/artifact-proxy/kustomization.yaml b/.github/resources/manifests/multiuser/artifact-proxy/kustomization.yaml index ab69b446b7b..870657129e7 100644 --- a/.github/resources/manifests/multiuser/artifact-proxy/kustomization.yaml +++ b/.github/resources/manifests/multiuser/artifact-proxy/kustomization.yaml @@ -46,6 +46,10 @@ patches: target: kind: Deployment name: ml-pipeline + - path: ../../base/driver-plugin-cm-path.yaml + target: + kind: ConfigMap + name: ml-pipeline-driver-agent - path: ../../base/grpc-specs.yaml target: kind: Deployment diff --git a/.github/resources/manifests/multiuser/cache-disabled/kustomization.yaml b/.github/resources/manifests/multiuser/cache-disabled/kustomization.yaml index 72b3b6e2767..b8411b2441c 100644 --- a/.github/resources/manifests/multiuser/cache-disabled/kustomization.yaml +++ b/.github/resources/manifests/multiuser/cache-disabled/kustomization.yaml @@ -46,6 +46,10 @@ patches: target: kind: Deployment name: ml-pipeline + - path: ../../base/driver-plugin-cm-path.yaml + target: + kind: ConfigMap + name: ml-pipeline-driver-agent - path: cache-env.yaml target: kind: Deployment diff --git a/.github/resources/manifests/multiuser/default/kustomization.yaml b/.github/resources/manifests/multiuser/default/kustomization.yaml index f3b9e8345bc..1a7c4d23997 100644 --- a/.github/resources/manifests/multiuser/default/kustomization.yaml +++ b/.github/resources/manifests/multiuser/default/kustomization.yaml @@ -46,6 +46,10 @@ patches: target: kind: Deployment name: ml-pipeline + - path: ../../base/driver-plugin-cm-path.yaml + target: + kind: ConfigMap + name: ml-pipeline-driver-agent - path: ../../base/grpc-specs.yaml target: kind: Deployment diff --git a/.github/resources/manifests/standalone/cache-disabled-proxy/kustomization.yaml b/.github/resources/manifests/standalone/cache-disabled-proxy/kustomization.yaml index 3e41d8ee530..f8950444cf1 100644 --- a/.github/resources/manifests/standalone/cache-disabled-proxy/kustomization.yaml +++ b/.github/resources/manifests/standalone/cache-disabled-proxy/kustomization.yaml @@ -9,3 +9,7 @@ patches: target: kind: Deployment name: ml-pipeline + - path: ../../base/driver-plugin-cm-path.yaml + target: + kind: ConfigMap + name: ml-pipeline-driver-agent diff --git a/.github/resources/manifests/standalone/cache-disabled/kustomization.yaml b/.github/resources/manifests/standalone/cache-disabled/kustomization.yaml index 9d3ca6e1be8..67969dcbb98 100644 --- a/.github/resources/manifests/standalone/cache-disabled/kustomization.yaml +++ b/.github/resources/manifests/standalone/cache-disabled/kustomization.yaml @@ -42,6 +42,10 @@ patches: target: kind: Deployment name: ml-pipeline + - path: ../../base/driver-plugin-cm-path.yaml + target: + kind: ConfigMap + name: ml-pipeline-driver-agent - path: cache-env.yaml target: kind: Deployment diff --git a/.github/resources/manifests/standalone/default/kustomization.yaml b/.github/resources/manifests/standalone/default/kustomization.yaml index fb78a97ff27..154f120843f 100644 --- a/.github/resources/manifests/standalone/default/kustomization.yaml +++ b/.github/resources/manifests/standalone/default/kustomization.yaml @@ -42,6 +42,10 @@ patches: target: kind: Deployment name: ml-pipeline + - path: ../../base/driver-plugin-cm-path.yaml + target: + kind: ConfigMap + name: ml-pipeline-driver-agent - path: ../../base/grpc-specs.yaml target: kind: Deployment diff --git a/.github/resources/manifests/standalone/proxy/kustomization.yaml b/.github/resources/manifests/standalone/proxy/kustomization.yaml index 6904ae9b5b6..4354a54b55e 100644 --- a/.github/resources/manifests/standalone/proxy/kustomization.yaml +++ b/.github/resources/manifests/standalone/proxy/kustomization.yaml @@ -46,6 +46,10 @@ patches: target: kind: Deployment name: ml-pipeline + - path: ../../base/driver-plugin-cm-path.yaml + target: + kind: ConfigMap + name: ml-pipeline-driver-agent - path: ../../base/grpc-specs.yaml target: kind: Deployment diff --git a/.github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml b/.github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml new file mode 100644 index 00000000000..0b83857803c --- /dev/null +++ b/.github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml @@ -0,0 +1,45 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ml-pipeline-driver-agent +data: + sidecar.container: | + name: driver-plugin + image: kind-registry:5000/driver:ci + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8080 + env: + - name: LOG_ACCESS_KEY + valueFrom: + secretKeyRef: + name: mlpipeline-minio-artifact + key: accesskey + - name: LOG_SECRET_KEY + valueFrom: + secretKeyRef: + name: mlpipeline-minio-artifact + key: secretkey + resources: + requests: + cpu: "0.1" + memory: "64Mi" + limits: + cpu: "0.5" + memory: "0.5Gi" + securityContext: + runAsNonRoot: true + runAsUser: 65534 + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: argo-workflows-agent-ca-certificates + mountPath: /kfp/certs + readOnly: true + - name: var-run-argo + mountPath: /kfp/log + readOnly: false diff --git a/.github/resources/manifests/standalone/tls-enabled/kustomization.yaml b/.github/resources/manifests/standalone/tls-enabled/kustomization.yaml index 7ba52e6fb93..4ca6bbe1b96 100644 --- a/.github/resources/manifests/standalone/tls-enabled/kustomization.yaml +++ b/.github/resources/manifests/standalone/tls-enabled/kustomization.yaml @@ -46,6 +46,10 @@ patches: target: kind: Deployment name: ml-pipeline + - path: driver-plugin-cm-path.yaml + target: + kind: ConfigMap + name: ml-pipeline-driver-agent - path: ../../base/grpc-specs.yaml target: kind: Deployment diff --git a/.github/resources/scripts/collect-logs.sh b/.github/resources/scripts/collect-logs.sh index 89ea75324b1..15e83578924 100755 --- a/.github/resources/scripts/collect-logs.sh +++ b/.github/resources/scripts/collect-logs.sh @@ -29,6 +29,36 @@ function check_namespace { return 0 } +function describe_argo_workflows { + local NAMESPACE=$1 + echo "===== Argo Workflows Inspection =====" + for wf in $(kubectl get wf -n "$NAMESPACE" -o json | jq -r '.items[] | select(.status.phase=="Failed" or .status.phase=="Running") | .metadata.name'); do + echo "Inspected workflow: $wf" + kubectl get wf "$wf" -n "$NAMESPACE" || true + pods=$(kubectl get po -n "$NAMESPACE" -l "workflows.argoproj.io/workflow=$wf" -o jsonpath='{.items[*].metadata.name}') + for pod in $pods; do + phase=$(kubectl get po "$pod" -n "$NAMESPACE" -o jsonpath='{.status.phase}') + echo "Inspect Pod: $pod, Status: $phase" + if [[ "$phase" != "Pending" && "$phase" != "Succeeded" ]]; then + echo " ---> $pod Logs:" + if [[ "$pod" == *-agent ]]; then + kubectl logs "$pod" -n "$NAMESPACE" -c driver-plugin || true + else + kubectl logs "$pod" -n "$NAMESPACE" || true + fi + fi + echo " ---> Describe $pod:" + if [[ "$phase" != "Succeeded" ]]; then + echo " ---> Describe:" + kubectl describe po "$pod" -n "$NAMESPACE" + fi + done + done + echo "===== Argo Workflows data =====" + kubectl get events -n "${NAMESPACE}" --field-selector involvedObject.kind=Workflow --sort-by='.metadata.creationTimestamp' + echo "===============================" +} + function display_pod_info { local NAMESPACE=$1 @@ -52,7 +82,13 @@ function display_pod_info { kubectl describe pod "${POD_NAME}" -n "${NAMESPACE}" | grep -A 100 Events || echo "No events found for pod ${POD_NAME}." echo "----- LOGS -----" - kubectl logs "${POD_NAME}" -n "${NAMESPACE}" || echo "No logs found for pod ${POD_NAME}." + if [[ "${POD_NAME}" == *-agent* ]]; then + kubectl logs "${POD_NAME}" -n "${NAMESPACE}" -c driver-plugin || \ + echo "No logs found for pod ${POD_NAME}." + else + kubectl logs "${POD_NAME}" -n "${NAMESPACE}" || \ + echo "No logs found for pod ${POD_NAME}." + fi echo "===========================" echo "" @@ -64,6 +100,7 @@ function display_pod_info { if check_namespace "$NS"; then display_pod_info "$NS" + describe_argo_workflows "$NS" else exit 0 fi diff --git a/.github/resources/scripts/kfp-readiness/wait_for_pods.py b/.github/resources/scripts/kfp-readiness/wait_for_pods.py index fc67d5fda0e..fc604586585 100644 --- a/.github/resources/scripts/kfp-readiness/wait_for_pods.py +++ b/.github/resources/scripts/kfp-readiness/wait_for_pods.py @@ -30,7 +30,8 @@ def get_pod_statuses(): statuses = {} for pod in pods.items: pod_name = pod.metadata.name - if "system" not in pod_name: + # This filter is safe: 'ml-pipeline-persistenceagent-' will not be excluded and will be processed. + if not ("system" in pod_name or pod_name.endswith("-agent")): pod_status = pod.status.phase container_statuses = pod.status.container_statuses or [] ready = 0 diff --git a/.github/workflows/api-server-tests.yml b/.github/workflows/api-server-tests.yml index 154251c95b0..abcdda9bf51 100644 --- a/.github/workflows/api-server-tests.yml +++ b/.github/workflows/api-server-tests.yml @@ -114,7 +114,7 @@ jobs: shell: bash if: ${{ matrix.pod_to_pod_tls_enabled == 'true'}} run: | - kubectl get secret kfp-api-tls-cert -n kubeflow -o jsonpath='{.data.ca\.crt}' | base64 -d > "${{ github.workspace }}/ca.crt" + kubectl get secret argo-workflows-agent-ca-certificates -n kubeflow -o jsonpath='{.data.ca\.crt}' | base64 -d > "${{ github.workspace }}/ca.crt" echo "CA_CERT_PATH=${{ github.workspace }}/ca.crt" >> "$GITHUB_ENV" diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 4ac9ac66cbd..11bcf3c2e7d 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -113,7 +113,7 @@ jobs: shell: bash if: ${{ matrix.pod_to_pod_tls_enabled == 'true'}} run: | - kubectl get secret kfp-api-tls-cert -n kubeflow -o jsonpath='{.data.ca\.crt}' | base64 -d > "${{ github.workspace }}/ca.crt" + kubectl get secret argo-workflows-agent-ca-certificates -n kubeflow -o jsonpath='{.data.ca\.crt}' | base64 -d > "${{ github.workspace }}/ca.crt" echo "CA_CERT_PATH=${{ github.workspace }}/ca.crt" >> "$GITHUB_ENV" - name: Configure Input Variables shell: bash diff --git a/.github/workflows/legacy-v2-api-integration-tests.yml b/.github/workflows/legacy-v2-api-integration-tests.yml index 5ffd7fa34c1..35a4d2a6308 100644 --- a/.github/workflows/legacy-v2-api-integration-tests.yml +++ b/.github/workflows/legacy-v2-api-integration-tests.yml @@ -77,7 +77,7 @@ jobs: shell: bash if: ${{ matrix.pod_to_pod_tls_enabled == 'true' }} run: | - kubectl get secret kfp-api-tls-cert -n kubeflow -o jsonpath='{.data.ca\.crt}' | base64 -d > "${{ github.workspace }}/ca.crt" + kubectl get secret argo-workflows-agent-ca-certificates -n kubeflow -o jsonpath='{.data.ca\.crt}' | base64 -d > "${{ github.workspace }}/ca.crt" echo "CA_CERT_PATH=${{ github.workspace }}/ca.crt" >> "$GITHUB_ENV" - name: Forward MLMD port diff --git a/backend/Dockerfile.driver b/backend/Dockerfile.driver index f57af8c0e4c..df4351e5cad 100644 --- a/backend/Dockerfile.driver +++ b/backend/Dockerfile.driver @@ -27,7 +27,7 @@ RUN GO111MODULE=on go mod download COPY . . -RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux go build -tags netgo -gcflags="${GCFLAGS}" -ldflags '-extldflags "-static"' -o /bin/driver ./backend/src/v2/cmd/driver/*.go +RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux go build -tags netgo -gcflags="${GCFLAGS}" -ldflags '-extldflags "-static"' -o /bin/driver ./backend/src/driver/*.go FROM alpine:3.21 diff --git a/backend/src/common/util/context_logger.go b/backend/src/common/util/context_logger.go new file mode 100644 index 00000000000..0240e819dde --- /dev/null +++ b/backend/src/common/util/context_logger.go @@ -0,0 +1,73 @@ +package util + +import ( + "context" + "fmt" + "io" + "os" + + "github.com/sirupsen/logrus" +) + +type CtxKey string + +const ( + contextLoggerKey CtxKey = "driver_log_key" +) + +func newFileLogger(logFile string) (*logrus.Logger, io.Closer, error) { + f, err := os.Create(logFile) + if err != nil { + return nil, nil, err + } + + logger := logrus.New() + logger.Out = io.MultiWriter(os.Stdout, f) + logger.Formatter = &logrus.TextFormatter{} + return logger, f, nil +} + +// WithExistingLogger For testing only +func WithExistingLogger(ctx context.Context, logger *logrus.Logger) context.Context { + return context.WithValue(ctx, contextLoggerKey, logger) +} + +func WithLogger(ctx context.Context, logFile string) (context.Context, io.Closer, error) { + if ctx == nil { + return nil, nil, fmt.Errorf( + "error during creation of the logger for logId: %v. ctx can not be nil", + logFile, + ) + } + + if GetLoggerFrom(ctx) != nil { + return nil, nil, fmt.Errorf("logger already exists in context") + } + + logger, f, err := newFileLogger(logFile) + if err != nil { + return nil, nil, fmt.Errorf( + "error during creation of the logger for logId: %v details: %w", + logFile, + err, + ) + } + + ctx = context.WithValue(ctx, contextLoggerKey, logger) + + return ctx, f, nil +} + +func GetLoggerFrom(ctx context.Context) *logrus.Logger { + v := ctx.Value(contextLoggerKey) + if v == nil { + return nil + } + + logger, ok := v.(*logrus.Logger) + if !ok { + return nil + } + + return logger +} diff --git a/backend/src/driver/driverapi/request.go b/backend/src/driver/driverapi/request.go new file mode 100644 index 00000000000..60ea8d44e61 --- /dev/null +++ b/backend/src/driver/driverapi/request.go @@ -0,0 +1,69 @@ +// Copyright 2025 The Kubeflow Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package driverapi provides HTTP DTOs used by the driver server. +package driverapi + +type DriverPluginArgs struct { + CachedDecisionPath string `json:"cached_decision_path"` + Component string `json:"component,omitempty"` + Container string `json:"container,omitempty"` + DagExecutionID string `json:"dag_execution_id"` + IterationIndex string `json:"iteration_index"` + HTTPProxy string `json:"http_proxy"` + HTTPSProxy string `json:"https_proxy"` + NoProxy string `json:"no_proxy"` + KubernetesConfig string `json:"kubernetes_config,omitempty"` + RuntimeConfig string `json:"runtime_config,omitempty"` + PipelineName string `json:"pipeline_name"` + PublishLogs string `json:"publish_logs,omitempty"` + RunID string `json:"run_id"` + RunName string `json:"run_name"` + RunDisplayName string `json:"run_display_name"` + TaskName string `json:"task_name"` + Task string `json:"task"` + Type string `json:"type"` + CacheDisabledFlag bool `json:"cache_disabled"` + ExecutionIDPath string `json:"execution_id_path"` + IterationCountPath string `json:"iteration_count_path"` + ConditionPath string `json:"condition_path"` + PodSpecPatchPath string `json:"pod_spec_patch_path"` + MLMDServerAddress string `json:"mlmd_server_address"` + MLMDServerPort string `json:"mlmd_server_port"` + MlPipelineServerAddress string `json:"ml_pipeline_server_address"` + MlPipelineServerPort string `json:"ml_pipeline_server_port"` + MlPipelineTLSEnabled bool `json:"ml_pipeline_tls_enabled"` + MetadataTLSEnabled bool `json:"metadata_tls_enabled"` + CACertPath string `json:"ca_cert_path"` + LogLevel string `json:"log_level"` + DefaultRunAsUser *int64 `json:"default_run_as_user,omitempty"` + DefaultRunAsGroup *int64 `json:"default_run_as_group,omitempty"` + DefaultRunAsNonRoot string `json:"default_run_as_non_root,omitempty"` +} + +type DriverPlugin struct { + DriverPlugin *DriverPluginContainer `json:"driver-plugin"` +} + +type DriverPluginContainer struct { + Args *DriverPluginArgs `json:"args"` +} + +type DriverTemplate struct { + Plugin *DriverPlugin `json:"plugin"` +} + +type DriverRequest struct { + Template *DriverTemplate `json:"template"` +} diff --git a/backend/src/driver/driverapi/response.go b/backend/src/driver/driverapi/response.go new file mode 100644 index 00000000000..3656138cf97 --- /dev/null +++ b/backend/src/driver/driverapi/response.go @@ -0,0 +1,34 @@ +// Copyright 2025 The Kubeflow Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package driverapi + +type DriverResponse struct { + Node Node `json:"node"` +} + +type Node struct { + Phase string `json:"phase"` + Outputs Outputs `json:"outputs"` + Message string `json:"message"` +} + +type Outputs struct { + Parameters []Parameter `json:"parameters"` +} + +type Parameter struct { + Name string `json:"name"` + Value string `json:"value"` +} diff --git a/backend/src/driver/main.go b/backend/src/driver/main.go new file mode 100644 index 00000000000..4d851735522 --- /dev/null +++ b/backend/src/driver/main.go @@ -0,0 +1,90 @@ +// Copyright 2021-2023 The Kubeflow Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "flag" + "fmt" + "net/http" + + "github.com/kubeflow/pipelines/backend/src/common/util" + + "github.com/golang/glog" + "github.com/kubeflow/pipelines/backend/src/v2/metadata" + "github.com/kubeflow/pipelines/kubernetes_platform/go/kubernetesplatform" +) + +const ( + unsetProxyArgValue = "unset" + RootDag = "ROOT_DAG" + DAG = "DAG" + CONTAINER = "CONTAINER" +) + +var ( + logLevel = flag.String("log_level", "1", "The verbosity level to log.") + serverPort = flag.String("server_port", ":8080", "Server port") +) + +func main() { + flag.Parse() + + glog.Infof("Setting log level to: '%s'", *logLevel) + err := flag.Set("v", *logLevel) + if err != nil { + glog.Warningf("Failed to set log level: %s", err.Error()) + } + + http.HandleFunc("/api/v1/template.execute", ExecutePlugin) + glog.Infof("Server started at http://localhost%v", *serverPort) + err = http.ListenAndServe(*serverPort, nil) + if err != nil { + glog.Warningf("Failed to start http server: %s", err.Error()) + } +} + +// Use WARNING default logging level to facilitate troubleshooting. +func init() { + flag.Set("logtostderr", "true") + // Change the WARNING to INFO level for debugging. + flag.Set("stderrthreshold", "WARNING") +} + +func parseExecConfigJSON(k8sExecConfigJSON *string) (*kubernetesplatform.KubernetesExecutorConfig, error) { + var k8sExecCfg *kubernetesplatform.KubernetesExecutorConfig + if *k8sExecConfigJSON != "" { + k8sExecCfg = &kubernetesplatform.KubernetesExecutorConfig{} + if err := util.UnmarshalString(*k8sExecConfigJSON, k8sExecCfg); err != nil { + return nil, fmt.Errorf("failed to unmarshal Kubernetes config, error: %w\nKubernetesConfig: %v", err, k8sExecConfigJSON) + } + } + return k8sExecCfg, nil +} + +func prettyPrint(jsonStr string) string { + var prettyJSON bytes.Buffer + err := json.Indent(&prettyJSON, []byte(jsonStr), "", " ") + if err != nil { + return jsonStr + } + return prettyJSON.String() +} + +func newMlmdClient(mlmdServerAddress string, mlmdServerPort string, tlsCfg *tls.Config) (*metadata.Client, error) { + return metadata.NewClient(mlmdServerAddress, mlmdServerPort, tlsCfg) +} diff --git a/backend/src/driver/main_test.go b/backend/src/driver/main_test.go new file mode 100644 index 00000000000..8619ad44f47 --- /dev/null +++ b/backend/src/driver/main_test.go @@ -0,0 +1,106 @@ +// Copyright 2025 The Kubeflow Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "testing" + + "github.com/kubeflow/pipelines/backend/src/driver/driverapi" + "github.com/kubeflow/pipelines/backend/src/v2/driver" + "github.com/kubeflow/pipelines/kubernetes_platform/go/kubernetesplatform" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" +) + +func strPtr(s string) *string { + return &s +} + +func TestSpecParsing(t *testing.T) { + tt := []struct { + name string + input *string + expected *kubernetesplatform.KubernetesExecutorConfig + wantErr bool + }{ + { + "Valid - test kubecfg value parse.", + strPtr("{\"imagePullSecret\":[{\"secret_name\":\"value1\"}]}"), + &kubernetesplatform.KubernetesExecutorConfig{ + ImagePullSecret: []*kubernetesplatform.ImagePullSecret{ + {SecretName: "value1"}, + }, + }, + false, + }, + { + "Valid - test kubecfg value ignores unknown field.", + strPtr("{\"imagePullSecret\":[{\"secret_name\":\"value1\"}], \"unknown_field\": \"something\"}"), + &kubernetesplatform.KubernetesExecutorConfig{ + ImagePullSecret: []*kubernetesplatform.ImagePullSecret{ + {SecretName: "value1"}, + }, + }, + false, + }, + } + + for _, tc := range tt { + t.Logf("Running test case: %s", tc.name) + cfg, err := parseExecConfigJSON(tc.input) + assert.Equal(t, tc.wantErr, err != nil) + assert.True(t, proto.Equal(tc.expected, cfg)) + } +} + +func Test_extractOutputParametersContainer(t *testing.T) { + execution := &driver.Execution{} + + outputs := extractOutputParameters(execution, CONTAINER) + + require.NotEmpty(t, outputs) + verifyOutputParameter(t, outputs, "condition", "nil") +} + +func Test_extractOutputParametersRootDAG(t *testing.T) { + execution := &driver.Execution{} + + outputs := extractOutputParameters(execution, RootDag) + require.NotEmpty(t, outputs) + verifyOutputParameter(t, outputs, "iteration-count", "0") + verifyOutputParameter(t, outputs, "condition", "nil") +} + +func Test_extractOutputParametersDAG(t *testing.T) { + execution := &driver.Execution{} + + outputs := extractOutputParameters(execution, DAG) + + require.NotEmpty(t, outputs) + verifyOutputParameter(t, outputs, "iteration-count", "0") + verifyOutputParameter(t, outputs, "condition", "nil") +} + +func verifyOutputParameter(t *testing.T, parameters []driverapi.Parameter, key, expectedValue string) { + filtered := make([]driverapi.Parameter, 0, 1) + for _, p := range parameters { + if p.Name == key { + filtered = append(filtered, p) + } + } + require.Len(t, filtered, 1) + require.Equal(t, expectedValue, filtered[0].Value) +} diff --git a/backend/src/driver/rpc_handler.go b/backend/src/driver/rpc_handler.go new file mode 100644 index 00000000000..812cf7cc7ec --- /dev/null +++ b/backend/src/driver/rpc_handler.go @@ -0,0 +1,475 @@ +// Copyright 2025 The Kubeflow Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "strconv" + "strings" + "time" + + "github.com/golang/glog" + "github.com/google/uuid" + "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" + "github.com/kubeflow/pipelines/backend/src/apiserver/config/proxy" + "github.com/kubeflow/pipelines/backend/src/common/util" + "github.com/kubeflow/pipelines/backend/src/driver/driverapi" + "github.com/kubeflow/pipelines/backend/src/v2/cacheutils" + "github.com/kubeflow/pipelines/backend/src/v2/config" + "github.com/kubeflow/pipelines/backend/src/v2/driver" + "github.com/kubeflow/pipelines/backend/src/v2/metadata" + "github.com/kubeflow/pipelines/backend/src/v2/objectstore" + "google.golang.org/protobuf/encoding/protojson" + "k8s.io/client-go/kubernetes" +) + +type driverLogArtifactContext struct { + Execution *driver.Execution + Task string + LocalPath string + OutputPathPrefix string + Namespace string + PipelineRoot string + StoreSessionInfo string + LogID string +} + +func ExecutePlugin(w http.ResponseWriter, r *http.Request) { + defer func(Body io.ReadCloser) { + err := Body.Close() + if err != nil { + glog.Errorf("Error closing response body: %v", err) + } + }(r.Body) + + if r.Method != http.MethodPost { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + glog.Infof("Received request to execute plugin: %v", r) + args, err := parseDriverRequestArgs(r) + if err != nil { + glog.Errorf("Failed to parse driver request args: %v", err) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if args == nil { + glog.Errorf("Failed to parse driver request args: nil") + http.Error(w, "Driver plugin requires at least one argument", http.StatusBadRequest) + return + } + execution, err := drive(*args) + outputs := extractOutputParameters(execution, args.Type) + if err != nil { + glog.Errorf("unable to drive execution: %v", err) + resp := driverapi.DriverResponse{ + Node: driverapi.Node{ + Phase: "Failed", + Outputs: driverapi.Outputs{ + Parameters: outputs, + }, + Message: fmt.Sprintf("unable to drive execution: %v", err), + }, + } + WriteJSONResponse(w, resp) + return + } + if execution != nil && execution.ExecutorInput != nil { + executorInputBytes, err := protojson.Marshal(execution.ExecutorInput) + if err != nil { + WriteJSONResponse(w, driverapi.DriverResponse{ + Node: driverapi.Node{ + Phase: "Failed", + Outputs: driverapi.Outputs{ + Parameters: outputs, + }, + Message: fmt.Sprintf("unable to drive execution: failed to marshal ExecutorInput to JSON: %v", err), + }, + }) + return + } + executorInputJSON := string(executorInputBytes) + glog.Infof("output ExecutorInput:%s\n", prettyPrint(executorInputJSON)) + } + resp := driverapi.DriverResponse{ + Node: driverapi.Node{ + Phase: "Succeeded", + Outputs: driverapi.Outputs{ + Parameters: outputs, + }, + }, + } + WriteJSONResponse(w, resp) +} + +func parseDriverRequestArgs(r *http.Request) (*driverapi.DriverPluginArgs, error) { + var body driverapi.DriverRequest + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + return nil, fmt.Errorf("failed to parse driver request body: %v", err) + } + switch { + case body.Template == nil: + return nil, fmt.Errorf("driver request body.Template is empty") + case body.Template.Plugin == nil: + return nil, fmt.Errorf("driver request body.Template.Plugin is empty") + case body.Template.Plugin.DriverPlugin == nil: + return nil, fmt.Errorf("driver request body.Template.Plugin.DriverPlugin is empty") + case body.Template.Plugin.DriverPlugin.Args == nil: + return nil, fmt.Errorf("driver request body.Template.Plugin.Args is empty") + } + args := body.Template.Plugin.DriverPlugin.Args + if err := validate(*args); err != nil { + return nil, err + } + return body.Template.Plugin.DriverPlugin.Args, nil +} + +func drive(args driverapi.DriverPluginArgs) (execution *driver.Execution, err error) { + defer func() { + if err != nil { + err = fmt.Errorf("KFP driver: %w", err) + } + }() + var ( + pipelineRoot string + storeSessionInfo string + namespace string + outputPathPrefix string + ) + var pipeline *metadata.Pipeline + logID := fmt.Sprintf("%d-%v-%v-%v", time.Now().UnixMilli(), args.IterationIndex, args.Type, args.TaskName) + logDir := "/kfp/log" + logFile := fmt.Sprintf("%s/%s.log", logDir, logID) + ctx, f, err := util.WithLogger(context.Background(), logFile) + if err != nil { + return nil, fmt.Errorf("failed to create driver logger: %v", err) + } + defer func() { + removeErr := os.Remove(logFile) + if removeErr != nil { + glog.Errorf("Failed to remove processed log file: %v", removeErr) + } + }() + defer func() { + if pipelineRoot != "" { + logContext := &driverLogArtifactContext{ + Execution: execution, + Task: args.TaskName, + LocalPath: logFile, + LogID: logID, + Namespace: namespace, + PipelineRoot: pipelineRoot, + StoreSessionInfo: storeSessionInfo, + OutputPathPrefix: outputPathPrefix, + } + uploadErr := uploadDriverLogArtifact(ctx, logContext) + if uploadErr != nil { + glog.Errorf("Failed to upload driver-logs artifact: %v", uploadErr) + } + } + }() + defer func() { + if f != nil { + closeErr := f.Close() + if closeErr != nil { + glog.Errorf("Failed to close file: %v", closeErr) + } + } + }() + + log := util.GetLoggerFrom(ctx) + + log.Infof("driver plugin arguments: %v", args) + // Support reading component spec from a file if value starts with @ + // This bypasses exec() argument size limits for large workflows + if strings.HasPrefix(args.Component, "@") { + filePath := (args.Component)[1:] // Remove the "@" prefix + data, err := os.ReadFile(filePath) + if err != nil { + return nil, fmt.Errorf("failed to read component spec from file %s: %w", filePath, err) + } + args.Component = string(data) + log.Infof("Read component spec from file: %s (%d bytes)", filePath, len(data)) + } + + proxy.InitializeConfig(args.HTTPProxy, args.HTTPSProxy, args.NoProxy) + + log.Infof("input ComponentSpec:%s\n", prettyPrint(args.Component)) + componentSpec := &pipelinespec.ComponentSpec{} + if err := util.UnmarshalString(args.Component, componentSpec); err != nil { + return nil, fmt.Errorf("failed to unmarshal component spec, error: %w\ncomponentSpec: %v", err, prettyPrint(args.Component)) + } + var taskSpec *pipelinespec.PipelineTaskSpec + if args.Task != "" { + log.Infof("input TaskSpec:%s\n", prettyPrint(args.Task)) + taskSpec = &pipelinespec.PipelineTaskSpec{} + if err := util.UnmarshalString(args.Task, taskSpec); err != nil { + return nil, fmt.Errorf("failed to unmarshal task spec, error: %w\ntask: %v", err, args.Task) + } + } + + containerSpec := &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{} + if args.Container != "" { + log.Infof("input ContainerSpec:%s\n", prettyPrint(args.Container)) + if err := util.UnmarshalString(args.Container, containerSpec); err != nil { + return nil, fmt.Errorf("failed to unmarshal container spec, error: %w\ncontainerSpec: %v", err, args.Container) + } + } + var runtimeConfig *pipelinespec.PipelineJob_RuntimeConfig + if args.RuntimeConfig != "" { + log.Infof("input RuntimeConfig:%s\n", prettyPrint(args.RuntimeConfig)) + runtimeConfig = &pipelinespec.PipelineJob_RuntimeConfig{} + if err := util.UnmarshalString(args.RuntimeConfig, runtimeConfig); err != nil { + return nil, fmt.Errorf("failed to unmarshal runtime config, error: %w\nruntimeConfig: %v", err, args.RuntimeConfig) + } + } + if args.KubernetesConfig != "" { + log.Infof("input kubernetesConfig:%s\n", prettyPrint(args.KubernetesConfig)) + } + k8sExecCfg, err := parseExecConfigJSON(&args.KubernetesConfig) + if err != nil { + return nil, err + } + namespace, err = config.InPodNamespace() + if err != nil { + return nil, err + } + var tlsCfg *tls.Config + if args.MetadataTLSEnabled { + tlsCfg, err = util.GetTLSConfig(args.CACertPath) + if err != nil { + return nil, fmt.Errorf("unable to drive driver: failed to load TLS configuration: %v", err) + } + } + client, err := newMlmdClient(args.MLMDServerAddress, args.MLMDServerPort, tlsCfg) + if err != nil { + return nil, err + } + cacheClient, err := cacheutils.NewClient(args.MlPipelineServerAddress, args.MlPipelineServerPort, args.CacheDisabledFlag, tlsCfg) + if err != nil { + return nil, err + } + + dagExecutionID, err := strconv.ParseInt(args.DagExecutionID, 10, 64) + if err != nil { + return nil, fmt.Errorf("failed to parse dag execution id, error: %w", err) + } + iterationIndex, err := strconv.Atoi(args.IterationIndex) + if err != nil { + return nil, fmt.Errorf("failed to parse iteration index, error: %w", err) + } + options := driver.Options{ + PipelineName: args.PipelineName, + RunID: args.RunID, + RunName: args.RunName, + RunDisplayName: args.RunDisplayName, + Namespace: namespace, + Component: componentSpec, + Task: taskSpec, + DAGExecutionID: dagExecutionID, + IterationIndex: iterationIndex, + PipelineLogLevel: args.LogLevel, + PublishLogs: args.PublishLogs, + CacheDisabled: args.CacheDisabledFlag, + DriverType: args.Type, + TaskName: args.TaskName, + MLPipelineServerAddress: args.MlPipelineServerAddress, + MLPipelineServerPort: args.MlPipelineServerPort, + MLPipelineTLSEnabled: args.MlPipelineTLSEnabled, + MLMDServerAddress: args.MLMDServerAddress, + MLMDServerPort: args.MLMDServerPort, + MLMDTLSEnabled: args.MetadataTLSEnabled, + CaCertPath: args.CACertPath, + } + + var driverErr error + switch args.Type { + case RootDag: + options.RuntimeConfig = runtimeConfig + execution, pipeline, driverErr = driver.RootDAG(ctx, options, client) + if driverErr != nil { + return nil, driverErr + } + pipelineRoot = pipeline.GetPipelineRoot() + storeSessionInfo = pipeline.GetStoreSessionInfo() + case DAG: + pipeline, driverErr = client.GetPipeline(ctx, options.PipelineName, options.RunID, "", "", "", "") + if driverErr != nil { + return nil, driverErr + } + pipelineRoot = pipeline.GetPipelineRoot() + storeSessionInfo = pipeline.GetStoreSessionInfo() + execution, driverErr = driver.DAG(ctx, pipeline, options, client) + case CONTAINER: + options.Container = containerSpec + options.KubernetesExecutorConfig = k8sExecCfg + if args.DefaultRunAsUser != nil && *args.DefaultRunAsUser >= 0 { + options.DefaultRunAsUser = args.DefaultRunAsUser + } + if args.DefaultRunAsGroup != nil && *args.DefaultRunAsGroup >= 0 { + options.DefaultRunAsGroup = args.DefaultRunAsGroup + } + if args.DefaultRunAsNonRoot != "" { + v, err := strconv.ParseBool(args.DefaultRunAsNonRoot) + if err == nil { + options.DefaultRunAsNonRoot = &v + } + } + pipeline, driverErr = client.GetPipeline(ctx, options.PipelineName, options.RunID, "", "", "", "") + if driverErr != nil { + return nil, driverErr + } + pipelineRoot = pipeline.GetPipelineRoot() + storeSessionInfo = pipeline.GetStoreSessionInfo() + outputPathPrefix = uuid.NewString() + execution, driverErr = driver.Container(ctx, pipeline, options, client, cacheClient, outputPathPrefix) + default: + err = fmt.Errorf("unknown driverType %s", args.Type) + } + if driverErr != nil { + log.Errorf("driver execution failed with error: %v", driverErr) + if execution == nil { + return nil, driverErr + } + defer func() { + // Override error with driver error, because driver error is more important. + // However, we continue running, because the following code prints debug info that + // may be helpful for figuring out why this failed. + err = driverErr + }() + } + + return execution, nil +} + +func uploadDriverLogArtifact(ctx context.Context, logContext *driverLogArtifactContext) error { + if logContext == nil { + return fmt.Errorf("logContext is nil") + } + if logContext.PipelineRoot != "" { + restConfig, err := util.GetKubernetesConfig() + if err != nil { + return fmt.Errorf("failed to get kubernetes config: %v", err) + } + k8sClient, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("failed to initialize kubernetes client set: %w", err) + } + session, err := objectstore.GetSessionInfoFromString(logContext.StoreSessionInfo) + if err != nil { + return fmt.Errorf("failed to get session info from store: %v", err) + } + bucketConfig, err := objectstore.ParseBucketConfig(logContext.PipelineRoot, session) + if err != nil { + return fmt.Errorf("failed to parse bucket config: %v", err) + } + bucket, err := objectstore.OpenBucket(ctx, k8sClient, logContext.Namespace, bucketConfig) + if err != nil { + return fmt.Errorf("failed to open bucket: %v", err) + } + key := fmt.Sprintf("driver/%s-logs", logContext.LogID) + if logContext.Execution != nil && logContext.OutputPathPrefix != "" { + key = fmt.Sprintf("%s/%s/driver-logs", logContext.Task, logContext.OutputPathPrefix) + } + glog.Infof("Uploading log key: %s ...", key) + err = objectstore.UploadBlob(ctx, bucket, logContext.LocalPath, key) + if err != nil { + return fmt.Errorf("failed to upload log: %v", err) + } + } + return nil +} + +func validate(args driverapi.DriverPluginArgs) error { + switch { + case args.Type == "": + return fmt.Errorf("argument type must be specified") + case args.HTTPProxy == unsetProxyArgValue: + return fmt.Errorf("argument http_proxy is required but can be an empty value") + case args.HTTPSProxy == unsetProxyArgValue: + return fmt.Errorf("argument https_proxy is required but can be an empty value") + case args.NoProxy == unsetProxyArgValue: + return fmt.Errorf("argument no_proxy is required but can be an empty value") + } + return nil +} + +func extractOutputParameters(execution *driver.Execution, driverType string) []driverapi.Parameter { + if execution == nil { + return []driverapi.Parameter{} + } + var outputs []driverapi.Parameter + if execution.ID != 0 { + outputs = append(outputs, driverapi.Parameter{ + Name: "execution-id", + Value: fmt.Sprint(execution.ID), + }) + } + if execution.IterationCount != nil { + outputs = append(outputs, driverapi.Parameter{ + Name: "iteration-count", + Value: fmt.Sprint(*execution.IterationCount), + }) + } else if driverType == RootDag || driverType == DAG { + outputs = append(outputs, driverapi.Parameter{ + Name: "iteration-count", + Value: "0", + }) + } + if execution.Cached != nil { + outputs = append(outputs, driverapi.Parameter{ + Name: "cached-decision", + Value: strconv.FormatBool(*execution.Cached), + }) + } + if execution.Condition != nil { + outputs = append(outputs, driverapi.Parameter{ + Name: "condition", + Value: strconv.FormatBool(*execution.Condition), + }) + } else if driverType == DAG || driverType == RootDag || driverType == CONTAINER { + // nil is a valid value for Condition + outputs = append(outputs, driverapi.Parameter{ + Name: "condition", + Value: "nil", + }) + } + if execution.PodSpecPatch != "" { + outputs = append(outputs, driverapi.Parameter{ + Name: "pod-spec-patch", + Value: execution.PodSpecPatch, + }) + } else { + outputs = append(outputs, driverapi.Parameter{ + Name: "pod-spec-patch", + Value: "", + }) + } + return outputs +} + +func WriteJSONResponse(w http.ResponseWriter, payload driverapi.DriverResponse) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + if err := json.NewEncoder(w).Encode(payload); err != nil { + http.Error(w, "failed to encode response", http.StatusInternalServerError) + } +} diff --git a/backend/src/v2/cmd/driver/execution_paths.go b/backend/src/v2/cmd/driver/execution_paths.go deleted file mode 100644 index 584d29065d5..00000000000 --- a/backend/src/v2/cmd/driver/execution_paths.go +++ /dev/null @@ -1,9 +0,0 @@ -package main - -type ExecutionPaths struct { - ExecutionID string - IterationCount string - CachedDecision string - Condition string - PodSpecPatch string -} diff --git a/backend/src/v2/cmd/driver/main.go b/backend/src/v2/cmd/driver/main.go deleted file mode 100644 index 428832e6bea..00000000000 --- a/backend/src/v2/cmd/driver/main.go +++ /dev/null @@ -1,384 +0,0 @@ -// Copyright 2021-2023 The Kubeflow Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package main - -import ( - "bytes" - "context" - "crypto/tls" - "encoding/json" - "flag" - "fmt" - - "google.golang.org/protobuf/encoding/protojson" - - "github.com/kubeflow/pipelines/backend/src/apiserver/config/proxy" - "github.com/kubeflow/pipelines/backend/src/common/util" - - "os" - "path/filepath" - "strconv" - "strings" - - "github.com/golang/glog" - "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" - "github.com/kubeflow/pipelines/backend/src/v2/cacheutils" - "github.com/kubeflow/pipelines/backend/src/v2/config" - "github.com/kubeflow/pipelines/backend/src/v2/driver" - "github.com/kubeflow/pipelines/backend/src/v2/metadata" - "github.com/kubeflow/pipelines/kubernetes_platform/go/kubernetesplatform" -) - -const ( - driverTypeArg = "type" - httpProxyArg = "http_proxy" - httpsProxyArg = "https_proxy" - noProxyArg = "no_proxy" - unsetProxyArgValue = "unset" - ROOT_DAG = "ROOT_DAG" - DAG = "DAG" - CONTAINER = "CONTAINER" -) - -var ( - // inputs - driverType = flag.String(driverTypeArg, "", "task driver type, one of ROOT_DAG, DAG, CONTAINER") - pipelineName = flag.String("pipeline_name", "", "pipeline context name") - runID = flag.String("run_id", "", "pipeline run uid") - runName = flag.String("run_name", "", "pipeline run name (Kubernetes object name)") - runDisplayName = flag.String("run_display_name", "", "pipeline run display name") - componentSpecJson = flag.String("component", "{}", "component spec") - taskSpecJson = flag.String("task", "", "task spec") - runtimeConfigJson = flag.String("runtime_config", "", "jobruntime config") - iterationIndex = flag.Int("iteration_index", -1, "iteration index, -1 means not an interation") - taskName = flag.String("task_name", "", "original task name, used for proper input resolution in the container/dag driver") - - // container inputs - dagExecutionID = flag.Int64("dag_execution_id", 0, "DAG execution ID") - containerSpecJson = flag.String("container", "{}", "container spec") - k8sExecConfigJson = flag.String("kubernetes_config", "{}", "kubernetes executor config") - - // config - mlPipelineServerAddress = flag.String("ml_pipeline_server_address", "ml-pipeline", "The name of the ML pipeline API server address.") - mlPipelineServerPort = flag.String("ml_pipeline_server_port", "8887", "The port of the ML pipeline API server.") - mlmdServerAddress = flag.String("mlmd_server_address", "", "MLMD server address") - mlmdServerPort = flag.String("mlmd_server_port", "", "MLMD server port") - - // output paths - executionIDPath = flag.String("execution_id_path", "", "Exeucution ID output path") - iterationCountPath = flag.String("iteration_count_path", "", "Iteration Count output path") - podSpecPatchPath = flag.String("pod_spec_patch_path", "", "Pod Spec Patch output path") - // the value stored in the paths will be either 'true' or 'false' - cachedDecisionPath = flag.String("cached_decision_path", "", "Cached Decision output path") - conditionPath = flag.String("condition_path", "", "Condition output path") - logLevel = flag.String("log_level", "1", "The verbosity level to log.") - - // proxy - httpProxy = flag.String(httpProxyArg, unsetProxyArgValue, "The proxy for HTTP connections.") - httpsProxy = flag.String(httpsProxyArg, unsetProxyArgValue, "The proxy for HTTPS connections.") - noProxy = flag.String(noProxyArg, unsetProxyArgValue, "Addresses that should ignore the proxy.") - publishLogs = flag.String("publish_logs", "true", "Whether to publish component logs to the object store") - cacheDisabledFlag = flag.Bool("cache_disabled", false, "Disable cache globally.") - mlPipelineTLSEnabled = flag.Bool("ml_pipeline_tls_enabled", false, "Set to true if mlpipeline API server serves over TLS.") - metadataTLSEnabled = flag.Bool("metadata_tls_enabled", false, "Set to true if MLMD serves over TLS.") - caCertPath = flag.String("ca_cert_path", "", "The path to the CA certificate to trust on connections to the ML pipeline API server and metadata server.") - defaultRunAsUser = flag.Int64("default_run_as_user", -1, "Admin-configured default runAsUser for user containers. -1 means not set.") - defaultRunAsGroup = flag.Int64("default_run_as_group", -1, "Admin-configured default runAsGroup for user containers. -1 means not set.") - defaultRunAsNonRoot = flag.String("default_run_as_non_root", "", "Admin-configured default runAsNonRoot for user containers. Empty means not set.") -) - -// func RootDAG(pipelineName string, runID string, component *pipelinespec.ComponentSpec, task *pipelinespec.PipelineTaskSpec, mlmd *metadata.Client) (*Execution, error) { - -func main() { - flag.Parse() - - glog.Infof("Setting log level to: '%s'", *logLevel) - err := flag.Set("v", *logLevel) - if err != nil { - glog.Warningf("Failed to set log level: %s", err.Error()) - } - - err = drive() - if err != nil { - glog.Exitf("%v", err) - } -} - -// Use WARNING default logging level to facilitate troubleshooting. -func init() { - flag.Set("logtostderr", "true") - // Change the WARNING to INFO level for debugging. - flag.Set("stderrthreshold", "WARNING") -} - -func validate() error { - if *driverType == "" { - return fmt.Errorf("argument --%s must be specified", driverTypeArg) - } - if *httpProxy == unsetProxyArgValue { - return fmt.Errorf("argument --%s is required but can be an empty value", httpProxyArg) - } - if *httpsProxy == unsetProxyArgValue { - return fmt.Errorf("argument --%s is required but can be an empty value", httpsProxyArg) - } - if *noProxy == unsetProxyArgValue { - return fmt.Errorf("argument --%s is required but can be an empty value", noProxyArg) - } - // validation responsibility lives in driver itself, so we do not validate all other args - return nil -} - -func drive() (err error) { - defer func() { - if err != nil { - err = fmt.Errorf("KFP driver: %w", err) - } - }() - ctx := context.Background() - if err = validate(); err != nil { - return err - } - - // Support reading component spec from a file if value starts with @ - // This bypasses exec() argument size limits for large workflows - if strings.HasPrefix(*componentSpecJson, "@") { - filePath := (*componentSpecJson)[1:] // Remove the "@" prefix - data, err := os.ReadFile(filePath) - if err != nil { - return fmt.Errorf("failed to read component spec from file %s: %w", filePath, err) - } - *componentSpecJson = string(data) - glog.Infof("Read component spec from file: %s (%d bytes)", filePath, len(data)) - } - - proxy.InitializeConfig(*httpProxy, *httpsProxy, *noProxy) - glog.Infof("input ComponentSpec:%s\n", prettyPrint(*componentSpecJson)) - componentSpec := &pipelinespec.ComponentSpec{} - if err := util.UnmarshalString(*componentSpecJson, componentSpec); err != nil { - return fmt.Errorf("failed to unmarshal component spec, error: %w\ncomponentSpec: %v", err, prettyPrint(*componentSpecJson)) - } - var taskSpec *pipelinespec.PipelineTaskSpec - if *taskSpecJson != "" { - glog.Infof("input TaskSpec:%s\n", prettyPrint(*taskSpecJson)) - taskSpec = &pipelinespec.PipelineTaskSpec{} - if err := util.UnmarshalString(*taskSpecJson, taskSpec); err != nil { - return fmt.Errorf("failed to unmarshal task spec, error: %w\ntask: %v", err, taskSpecJson) - } - } - glog.Infof("input ContainerSpec:%s\n", prettyPrint(*containerSpecJson)) - containerSpec := &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{} - if err := util.UnmarshalString(*containerSpecJson, containerSpec); err != nil { - return fmt.Errorf("failed to unmarshal container spec, error: %w\ncontainerSpec: %v", err, containerSpecJson) - } - var runtimeConfig *pipelinespec.PipelineJob_RuntimeConfig - if *runtimeConfigJson != "" { - glog.Infof("input RuntimeConfig:%s\n", prettyPrint(*runtimeConfigJson)) - runtimeConfig = &pipelinespec.PipelineJob_RuntimeConfig{} - if err := util.UnmarshalString(*runtimeConfigJson, runtimeConfig); err != nil { - return fmt.Errorf("failed to unmarshal runtime config, error: %w\nruntimeConfig: %v", err, runtimeConfigJson) - } - } - k8sExecCfg, err := parseExecConfigJson(k8sExecConfigJson) - if err != nil { - return err - } - namespace, err := config.InPodNamespace() - if err != nil { - return err - } - var tlsCfg *tls.Config - if *metadataTLSEnabled { - tlsCfg, err = util.GetTLSConfig(*caCertPath) - if err != nil { - return err - } - } - client, err := newMlmdClient(*mlmdServerAddress, *mlmdServerPort, tlsCfg) - if err != nil { - return err - } - cacheClient, err := cacheutils.NewClient(*mlPipelineServerAddress, *mlPipelineServerPort, *cacheDisabledFlag, tlsCfg) - if err != nil { - return err - } - options := driver.Options{ - PipelineName: *pipelineName, - RunID: *runID, - RunName: *runName, - RunDisplayName: *runDisplayName, - Namespace: namespace, - Component: componentSpec, - Task: taskSpec, - DAGExecutionID: *dagExecutionID, - IterationIndex: *iterationIndex, - PipelineLogLevel: *logLevel, - PublishLogs: *publishLogs, - CacheDisabled: *cacheDisabledFlag, - DriverType: *driverType, - TaskName: *taskName, - MLPipelineServerAddress: *mlPipelineServerAddress, - MLPipelineServerPort: *mlPipelineServerPort, - MLMDServerAddress: *mlmdServerAddress, - MLMDServerPort: *mlmdServerPort, - MLPipelineTLSEnabled: *mlPipelineTLSEnabled, - MLMDTLSEnabled: *metadataTLSEnabled, - CaCertPath: *caCertPath, - } - var execution *driver.Execution - var driverErr error - switch *driverType { - case ROOT_DAG: - options.RuntimeConfig = runtimeConfig - execution, driverErr = driver.RootDAG(ctx, options, client) - case DAG: - execution, driverErr = driver.DAG(ctx, options, client) - case CONTAINER: - options.Container = containerSpec - options.KubernetesExecutorConfig = k8sExecCfg - // Set admin defaults only when explicitly configured (non-negative). - if *defaultRunAsUser >= 0 { - options.DefaultRunAsUser = defaultRunAsUser - } - if *defaultRunAsGroup >= 0 { - options.DefaultRunAsGroup = defaultRunAsGroup - } - if *defaultRunAsNonRoot != "" { - v, err := strconv.ParseBool(*defaultRunAsNonRoot) - if err == nil { - options.DefaultRunAsNonRoot = &v - } - } - execution, driverErr = driver.Container(ctx, options, client, cacheClient) - default: - err = fmt.Errorf("unknown driverType %s", *driverType) - } - if driverErr != nil { - if execution == nil { - return driverErr - } - defer func() { - // Override error with driver error, because driver error is more important. - // However, we continue running, because the following code prints debug info that - // may be helpful for figuring out why this failed. - err = driverErr - }() - } - - executionPaths := &ExecutionPaths{ - ExecutionID: *executionIDPath, - IterationCount: *iterationCountPath, - CachedDecision: *cachedDecisionPath, - Condition: *conditionPath, - PodSpecPatch: *podSpecPatchPath, - } - - return handleExecution(execution, *driverType, executionPaths) -} - -func parseExecConfigJson(k8sExecConfigJson *string) (*kubernetesplatform.KubernetesExecutorConfig, error) { - var k8sExecCfg *kubernetesplatform.KubernetesExecutorConfig - if *k8sExecConfigJson != "" { - glog.Infof("input kubernetesConfig:%s\n", prettyPrint(*k8sExecConfigJson)) - k8sExecCfg = &kubernetesplatform.KubernetesExecutorConfig{} - if err := util.UnmarshalString(*k8sExecConfigJson, k8sExecCfg); err != nil { - return nil, fmt.Errorf("failed to unmarshal Kubernetes config, error: %w\nKubernetesConfig: %v", err, k8sExecConfigJson) - } - } - return k8sExecCfg, nil -} - -func handleExecution(execution *driver.Execution, driverType string, executionPaths *ExecutionPaths) error { - if execution.ID != 0 { - glog.Infof("output execution.ID=%v", execution.ID) - if executionPaths.ExecutionID != "" { - if err := writeFile(executionPaths.ExecutionID, []byte(fmt.Sprint(execution.ID))); err != nil { - return fmt.Errorf("failed to write execution ID to file: %w", err) - } - } - } - if execution.IterationCount != nil { - if err := writeFile(executionPaths.IterationCount, []byte(fmt.Sprintf("%v", *execution.IterationCount))); err != nil { - return fmt.Errorf("failed to write iteration count to file: %w", err) - } - } else { - if driverType == ROOT_DAG || driverType == DAG { - if err := writeFile(executionPaths.IterationCount, []byte("0")); err != nil { - return fmt.Errorf("failed to write iteration count to file: %w", err) - } - } - } - if execution.Cached != nil { - if err := writeFile(executionPaths.CachedDecision, []byte(strconv.FormatBool(*execution.Cached))); err != nil { - return fmt.Errorf("failed to write cached decision to file: %w", err) - } - } - if execution.Condition != nil { - if err := writeFile(executionPaths.Condition, []byte(strconv.FormatBool(*execution.Condition))); err != nil { - return fmt.Errorf("failed to write condition to file: %w", err) - } - } else { - // nil is a valid value for Condition - if driverType == ROOT_DAG || driverType == DAG || driverType == CONTAINER { - if err := writeFile(executionPaths.Condition, []byte("nil")); err != nil { - return fmt.Errorf("failed to write condition to file: %w", err) - } - } - } - if execution.PodSpecPatch != "" { - glog.Infof("output podSpecPatch=\n%s\n", execution.PodSpecPatch) - if executionPaths.PodSpecPatch == "" { - return fmt.Errorf("--pod_spec_patch_path is required for container executor drivers") - } - if err := writeFile(executionPaths.PodSpecPatch, []byte(execution.PodSpecPatch)); err != nil { - return fmt.Errorf("failed to write pod spec patch to file: %w", err) - } - } - if execution.ExecutorInput != nil { - executorInputBytes, err := protojson.Marshal(execution.ExecutorInput) - if err != nil { - return fmt.Errorf("failed to marshal ExecutorInput to JSON: %w", err) - } - executorInputJSON := string(executorInputBytes) - glog.Infof("output ExecutorInput:%s\n", prettyPrint(executorInputJSON)) - } - return nil -} - -func prettyPrint(jsonStr string) string { - var prettyJSON bytes.Buffer - err := json.Indent(&prettyJSON, []byte(jsonStr), "", " ") - if err != nil { - return jsonStr - } - return prettyJSON.String() -} - -func writeFile(path string, data []byte) (err error) { - if path == "" { - return fmt.Errorf("path is not specified") - } - defer func() { - if err != nil { - err = fmt.Errorf("failed to write to %s: %w", path, err) - } - }() - if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { - return err - } - return os.WriteFile(path, data, 0o644) -} - -func newMlmdClient(mlmdServerAddress string, mlmdServerPort string, tlsCfg *tls.Config) (*metadata.Client, error) { - return metadata.NewClient(mlmdServerAddress, mlmdServerPort, tlsCfg) -} diff --git a/backend/src/v2/cmd/driver/main_test.go b/backend/src/v2/cmd/driver/main_test.go deleted file mode 100644 index 302600d2fee..00000000000 --- a/backend/src/v2/cmd/driver/main_test.go +++ /dev/null @@ -1,144 +0,0 @@ -package main - -import ( - "os" - "testing" - - "github.com/kubeflow/pipelines/backend/src/v2/driver" - "github.com/kubeflow/pipelines/kubernetes_platform/go/kubernetesplatform" - "github.com/stretchr/testify/assert" - "google.golang.org/protobuf/proto" -) - -func strPtr(s string) *string { - return &s -} - -func TestSpecParsing(t *testing.T) { - tt := []struct { - name string - input *string - expected *kubernetesplatform.KubernetesExecutorConfig - wantErr bool - }{ - { - "Valid - test kubecfg value parse.", - strPtr("{\"imagePullSecret\":[{\"secret_name\":\"value1\"}]}"), - &kubernetesplatform.KubernetesExecutorConfig{ - ImagePullSecret: []*kubernetesplatform.ImagePullSecret{ - {SecretName: "value1"}, - }, - }, - false, - }, - { - "Valid - test kubecfg value ignores unknown field.", - strPtr("{\"imagePullSecret\":[{\"secret_name\":\"value1\"}], \"unknown_field\": \"something\"}"), - &kubernetesplatform.KubernetesExecutorConfig{ - ImagePullSecret: []*kubernetesplatform.ImagePullSecret{ - {SecretName: "value1"}, - }, - }, - false, - }, - } - - for _, tc := range tt { - t.Logf("Running test case: %s", tc.name) - cfg, err := parseExecConfigJson(tc.input) - assert.Equal(t, tc.wantErr, err != nil) - assert.True(t, proto.Equal(tc.expected, cfg)) - } -} - -func Test_handleExecutionContainer(t *testing.T) { - execution := &driver.Execution{} - - executionPaths := &ExecutionPaths{ - Condition: "condition.txt", - } - - err := handleExecution(execution, CONTAINER, executionPaths) - - if err != nil { - t.Errorf("Unexpected error: %v", err) - } - - verifyFileContent(t, executionPaths.Condition, "nil") - - cleanup(t, executionPaths) -} - -func Test_handleExecutionRootDAG(t *testing.T) { - execution := &driver.Execution{} - - executionPaths := &ExecutionPaths{ - IterationCount: "iteration_count.txt", - Condition: "condition.txt", - } - - err := handleExecution(execution, ROOT_DAG, executionPaths) - - if err != nil { - t.Errorf("Unexpected error: %v", err) - } - - verifyFileContent(t, executionPaths.IterationCount, "0") - verifyFileContent(t, executionPaths.Condition, "nil") - - cleanup(t, executionPaths) -} - -func Test_handleExecutionDAG(t *testing.T) { - execution := &driver.Execution{} - - executionPaths := &ExecutionPaths{ - IterationCount: "iteration_count.txt", - Condition: "condition.txt", - } - - err := handleExecution(execution, DAG, executionPaths) - - if err != nil { - t.Errorf("Unexpected error: %v", err) - } - - verifyFileContent(t, executionPaths.IterationCount, "0") - verifyFileContent(t, executionPaths.Condition, "nil") - - cleanup(t, executionPaths) -} - -func cleanup(t *testing.T, executionPaths *ExecutionPaths) { - removeIfExists(t, executionPaths.IterationCount) - removeIfExists(t, executionPaths.ExecutionID) - removeIfExists(t, executionPaths.Condition) - removeIfExists(t, executionPaths.PodSpecPatch) - removeIfExists(t, executionPaths.CachedDecision) -} - -func removeIfExists(t *testing.T, filePath string) { - _, err := os.Stat(filePath) - if err == nil { - err = os.Remove(filePath) - if err != nil { - t.Errorf("Unexpected error while removing the created file: %v", err) - } - } -} - -func verifyFileContent(t *testing.T, filePath string, expectedContent string) { - _, err := os.Stat(filePath) - if os.IsNotExist(err) { - t.Errorf("Expected file %s to be created, but it doesn't exist", filePath) - } - - fileContent, err := os.ReadFile(filePath) - if err != nil { - t.Errorf("Failed to read file contents: %v", err) - } - - if string(fileContent) != expectedContent { - t.Errorf("Expected file fileContent to be %q, got %q", expectedContent, string(fileContent)) - } -} diff --git a/backend/src/v2/compiler/argocompiler/argo.go b/backend/src/v2/compiler/argocompiler/argo.go index 6032ae08ec0..eeb115b63bf 100644 --- a/backend/src/v2/compiler/argocompiler/argo.go +++ b/backend/src/v2/compiler/argocompiler/argo.go @@ -162,6 +162,7 @@ func Compile(jobArg *pipelinespec.PipelineJob, kubernetesSpecArg *pipelinespec.S PodMetadata: &wfapi.Metadata{ Annotations: map[string]string{ "pipelines.kubeflow.org/v2_component": "true", + util.AnnotationKeyIstioSidecarInject: util.AnnotationValueIstioSidecarInjectDisabled, }, Labels: map[string]string{ "pipelines.kubeflow.org/v2_component": "true", @@ -200,8 +201,6 @@ func Compile(jobArg *pipelinespec.PipelineJob, kubernetesSpecArg *pipelinespec.S // TODO(chensun): release process and update the images. launcherImage: GetLauncherImage(), launcherCommand: GetLauncherCommand(), - driverImage: GetDriverImage(), - driverCommand: GetDriverCommand(), job: job, spec: spec, executors: deploy.GetExecutors(), diff --git a/backend/src/v2/compiler/argocompiler/container.go b/backend/src/v2/compiler/argocompiler/container.go index ce4b89e4fac..62217364050 100644 --- a/backend/src/v2/compiler/argocompiler/container.go +++ b/backend/src/v2/compiler/argocompiler/container.go @@ -39,9 +39,7 @@ import ( const ( volumeNameKFPLauncher = "kfp-launcher" - volumeNameCABundle = "ca-bundle" LauncherImageEnvVar = "V2_LAUNCHER_IMAGE" - DriverImageEnvVar = "V2_DRIVER_IMAGE" // DefaultLauncherImage & DefaultDriverImage are set as latest here // but are overridden by environment variables set via k8s manifests. // For releases, the manifest will have the correct release version set. @@ -49,9 +47,6 @@ const ( DefaultLauncherImage = "ghcr.io/kubeflow/kfp-launcher:latest" LauncherCommandEnvVar = "V2_LAUNCHER_COMMAND" DefaultLauncherCommand = "launcher-v2" - DefaultDriverImage = "ghcr.io/kubeflow/kfp-driver:latest" - DefaultDriverCommand = "driver" - DriverCommandEnvVar = "V2_DRIVER_COMMAND" PipelineRunAsUserEnvVar = "PIPELINE_RUN_AS_USER" PipelineLogLevelEnvVar = "PIPELINE_LOG_LEVEL" PublishLogsEnvVar = "PUBLISH_LOGS" @@ -105,22 +100,6 @@ func GetLauncherImage() string { return launcherImage } -func GetDriverImage() string { - driverImage := os.Getenv(DriverImageEnvVar) - if driverImage == "" { - driverImage = DefaultDriverImage - } - return driverImage -} - -func GetDriverCommand() []string { - driverCommand := os.Getenv(DriverCommandEnvVar) - if driverCommand == "" { - driverCommand = DefaultDriverCommand - } - return strings.Split(driverCommand, " ") -} - func GetLauncherCommand() []string { launcherCommand := os.Getenv(LauncherCommandEnvVar) if launcherCommand == "" { @@ -148,10 +127,14 @@ func GetPipelineRunAsUser() *int64 { return &runAsUser } -func (c *workflowCompiler) containerDriverTask(name string, inputs containerDriverInputs) (*wfapi.DAGTask, *containerDriverOutputs) { +func (c *workflowCompiler) containerDriverTask(name string, inputs containerDriverInputs) (*wfapi.DAGTask, *containerDriverOutputs, error) { + template, err := c.addContainerDriverTemplate() + if err != nil { + return nil, nil, err + } dagTask := &wfapi.DAGTask{ Name: name, - Template: c.addContainerDriverTemplate(), + Template: template, Arguments: wfapi.Arguments{ Parameters: []wfapi.Parameter{ {Name: paramComponent, Value: wfapi.AnyStringPtr(inputs.component)}, @@ -179,71 +162,70 @@ func (c *workflowCompiler) containerDriverTask(name string, inputs containerDriv cached: taskOutputParameter(name, paramCachedDecision), condition: taskOutputParameter(name, paramCondition), } - return dagTask, outputs + return dagTask, outputs, nil } -func (c *workflowCompiler) addContainerDriverTemplate() string { +// Create the Argo Workflow executor plugin template for the container driver. +// See https://argo-workflows.readthedocs.io/en/latest/executor_plugins/ +func (c *workflowCompiler) addContainerDriverTemplate() (string, error) { name := "system-container-driver" _, ok := c.templates[name] if ok { - return name - } - - args := []string{ - "--type", "CONTAINER", - "--pipeline_name", c.spec.GetPipelineInfo().GetName(), - "--run_id", runID(), - "--run_name", runResourceName(), - "--run_display_name", c.job.DisplayName, - "--dag_execution_id", inputValue(paramParentDagID), - "--component", inputValue(paramComponent), - "--task", inputValue(paramTask), - "--task_name", inputValue(paramTaskName), - "--container", inputValue(paramContainer), - "--iteration_index", inputValue(paramIterationIndex), - "--cached_decision_path", outputPath(paramCachedDecision), - "--pod_spec_patch_path", outputPath(paramPodSpecPatch), - "--condition_path", outputPath(paramCondition), - "--kubernetes_config", inputValue(paramKubernetesConfig), - "--http_proxy", proxy.GetConfig().GetHttpProxy(), - "--https_proxy", proxy.GetConfig().GetHttpsProxy(), - "--no_proxy", proxy.GetConfig().GetNoProxy(), - "--ml_pipeline_server_address", config.GetMLPipelineServerConfig().Address, - "--ml_pipeline_server_port", config.GetMLPipelineServerConfig().Port, - "--mlmd_server_address", metadata.GetMetadataConfig().Address, - "--mlmd_server_port", metadata.GetMetadataConfig().Port, - } - if c.cacheDisabled { - args = append(args, "--cache_disabled") - } - if c.mlPipelineTLSEnabled { - args = append(args, "--ml_pipeline_tls_enabled") - } - if common.GetMetadataTLSEnabled() { - args = append(args, "--metadata_tls_enabled") + return name, nil + } + + args := map[string]interface{}{ + "type": "CONTAINER", + "pipeline_name": c.spec.GetPipelineInfo().GetName(), + "run_id": runID(), + "run_name": runResourceName(), + "run_display_name": c.job.DisplayName, + "dag_execution_id": inputValue(paramParentDagID), + "component": inputValue(paramComponent), + "task": inputValue(paramTask), + "task_name": inputValue(paramTaskName), + "container": inputValue(paramContainer), + "iteration_index": inputValue(paramIterationIndex), + "cached_decision_path": outputPath(paramCachedDecision), + "pod_spec_patch_path": outputPath(paramPodSpecPatch), + "condition_path": outputPath(paramCondition), + "kubernetes_config": inputValue(paramKubernetesConfig), + "http_proxy": proxy.GetConfig().GetHttpProxy(), + "https_proxy": proxy.GetConfig().GetHttpsProxy(), + "no_proxy": proxy.GetConfig().GetNoProxy(), + "ml_pipeline_server_address": config.GetMLPipelineServerConfig().Address, + "ml_pipeline_server_port": config.GetMLPipelineServerConfig().Port, + "mlmd_server_address": metadata.GetMetadataConfig().Address, + "mlmd_server_port": metadata.GetMetadataConfig().Port, + "cache_disabled": c.cacheDisabled, + "ml_pipeline_tls_enabled": c.mlPipelineTLSEnabled, + "metadata_tls_enabled": common.GetMetadataTLSEnabled(), } - setCABundle := false // If CABUNDLE_SECRET_NAME or CABUNDLE_CONFIGMAP_NAME is set, add ca_cert_path arg to container driver. if common.GetCaBundleSecretName() != "" || common.GetCaBundleConfigMapName() != "" { - args = append(args, "--ca_cert_path", common.CustomCaCertPath) - setCABundle = true + args["ca_cert_path"] = common.CustomCaCertPath } if value, ok := os.LookupEnv(PipelineLogLevelEnvVar); ok { - args = append(args, "--log_level", value) + args["log_level"] = value } if value, ok := os.LookupEnv(PublishLogsEnvVar); ok { - args = append(args, "--publish_logs", value) + args["publish_logs"] = value } if c.defaultRunAsUser != nil { - args = append(args, "--default_run_as_user", strconv.FormatInt(*c.defaultRunAsUser, 10)) + args["default_run_as_user"] = strconv.FormatInt(*c.defaultRunAsUser, 10) } if c.defaultRunAsGroup != nil { - args = append(args, "--default_run_as_group", strconv.FormatInt(*c.defaultRunAsGroup, 10)) + args["default_run_as_group"] = strconv.FormatInt(*c.defaultRunAsGroup, 10) } if c.defaultRunAsNonRoot != nil { - args = append(args, "--default_run_as_non_root", strconv.FormatBool(*c.defaultRunAsNonRoot)) + args["default_run_as_non_root"] = strconv.FormatBool(*c.defaultRunAsNonRoot) + } + + containerDriverPlugin, err := driverPlugin(args) + if err != nil { + return name, fmt.Errorf("failed to add container driver plugin: %v", err) } template := &wfapi.Template{ @@ -261,27 +243,17 @@ func (c *workflowCompiler) addContainerDriverTemplate() string { }, Outputs: wfapi.Outputs{ Parameters: []wfapi.Parameter{ - {Name: paramPodSpecPatch, ValueFrom: &wfapi.ValueFrom{Path: "/tmp/outputs/pod-spec-patch", Default: wfapi.AnyStringPtr("")}}, - {Name: paramCachedDecision, Default: wfapi.AnyStringPtr("false"), ValueFrom: &wfapi.ValueFrom{Path: "/tmp/outputs/cached-decision", Default: wfapi.AnyStringPtr("false")}}, - {Name: paramCondition, ValueFrom: &wfapi.ValueFrom{Path: "/tmp/outputs/condition", Default: wfapi.AnyStringPtr("true")}}, + {Name: paramPodSpecPatch, ValueFrom: &wfapi.ValueFrom{JSONPath: "$.pod-spec-patch", Default: wfapi.AnyStringPtr("")}}, + {Name: paramCachedDecision, Default: wfapi.AnyStringPtr("false"), ValueFrom: &wfapi.ValueFrom{JSONPath: "$.cached-decision", Default: wfapi.AnyStringPtr("false")}}, + {Name: paramCondition, ValueFrom: &wfapi.ValueFrom{JSONPath: "$.condition", Default: wfapi.AnyStringPtr("true")}}, }, }, - Container: &k8score.Container{ - Image: c.driverImage, - Command: c.driverCommand, - Args: args, - Resources: driverResources, - Env: append(proxy.GetConfig().GetEnvVars(), commonEnvs...), - }, + Plugin: containerDriverPlugin, } applySecurityContextToTemplate(template) - // If TLS is enabled (apiserver or metadata), add the custom CA bundle to the container driver template. - if setCABundle { - ConfigureCustomCABundle(template) - } c.templates[name] = template c.wf.Spec.Templates = append(c.wf.Spec.Templates, *template) - return name + return name, err } type containerExecutorInputs struct { diff --git a/backend/src/v2/compiler/argocompiler/container_test.go b/backend/src/v2/compiler/argocompiler/container_test.go index 1682874d790..0388015dff5 100644 --- a/backend/src/v2/compiler/argocompiler/container_test.go +++ b/backend/src/v2/compiler/argocompiler/container_test.go @@ -70,7 +70,12 @@ func TestAddContainerExecutorTemplate(t *testing.T) { } +// With the transition from Container to Plugin, environment variable configuration +// is now defined in manifests rather than in code, so we can no longer verify +// their behavior in a unit test. However, this is still fully tested within the +// workflow-compiler integration tests. func TestContainerDriverTemplate_IncludesKFPPodNameEnv(t *testing.T) { + t.Skip("Skipping test: env vars are now defined in manifests and cannot be verified in unit tests") proxy.InitializeConfigWithEmptyForTests() c := &workflowCompiler{ templates: make(map[string]*wfapi.Template), @@ -85,7 +90,8 @@ func TestContainerDriverTemplate_IncludesKFPPodNameEnv(t *testing.T) { job: &pipelinespec.PipelineJob{}, } - name := c.addContainerDriverTemplate() + name, err := c.addContainerDriverTemplate() + require.NoError(t, err) require.Equal(t, "system-container-driver", name) tmpl, exists := c.templates[name] diff --git a/backend/src/v2/compiler/argocompiler/dag.go b/backend/src/v2/compiler/argocompiler/dag.go index 658ea931fb2..da492928f85 100644 --- a/backend/src/v2/compiler/argocompiler/dag.go +++ b/backend/src/v2/compiler/argocompiler/dag.go @@ -19,15 +19,13 @@ import ( "sort" "strings" - "github.com/kubeflow/pipelines/backend/src/apiserver/config/proxy" - "github.com/kubeflow/pipelines/backend/src/v2/config" - "github.com/kubeflow/pipelines/backend/src/v2/metadata" - wfapi "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" "github.com/kubeflow/pipelines/backend/src/apiserver/common" + "github.com/kubeflow/pipelines/backend/src/apiserver/config/proxy" "github.com/kubeflow/pipelines/backend/src/v2/compiler" - k8score "k8s.io/api/core/v1" + "github.com/kubeflow/pipelines/backend/src/v2/config" + "github.com/kubeflow/pipelines/backend/src/v2/metadata" "k8s.io/apimachinery/pkg/util/intstr" ) @@ -297,7 +295,7 @@ func (c *workflowCompiler) task(name string, task *pipelinespec.PipelineTaskSpec // The following call will return an empty string for tasks without kubernetes-specific annotation. kubernetesConfigPlaceholder, _ := c.useKubernetesImpl(componentName) - driver, driverOutputs := c.containerDriverTask(driverTaskName, containerDriverInputs{ + driver, driverOutputs, err := c.containerDriverTask(driverTaskName, containerDriverInputs{ component: componentSpecPlaceholder, task: taskSpecJson, container: containerPlaceholder, @@ -306,6 +304,9 @@ func (c *workflowCompiler) task(name string, task *pipelinespec.PipelineTaskSpec kubernetesConfig: kubernetesConfigPlaceholder, taskName: effectiveTaskName, }) + if err != nil { + return nil, err + } if task.GetTriggerPolicy().GetCondition() == "" { driverOutputs.condition = "" } @@ -547,9 +548,13 @@ func (c *workflowCompiler) dagDriverTask(name string, inputs dagDriverInputs) (* Value: wfapi.AnyStringPtr(inputs.taskName), }) } + dagTemplate, err := c.addDAGDriverTemplate() + if err != nil { + return nil, nil, err + } t := &wfapi.DAGTask{ Name: name, - Template: c.addDAGDriverTemplate(), + Template: dagTemplate, Arguments: wfapi.Arguments{ Parameters: params, }, @@ -561,58 +566,56 @@ func (c *workflowCompiler) dagDriverTask(name string, inputs dagDriverInputs) (* }, nil } -func (c *workflowCompiler) addDAGDriverTemplate() string { +// Create the Argo Workflow executor plugin template for the dag driver. +// See https://argo-workflows.readthedocs.io/en/latest/executor_plugins/ +func (c *workflowCompiler) addDAGDriverTemplate() (string, error) { name := "system-dag-driver" _, ok := c.templates[name] if ok { - return name - } - - args := []string{ - "--type", inputValue(paramDriverType), - "--pipeline_name", c.spec.GetPipelineInfo().GetName(), - "--run_id", runID(), - "--run_name", runResourceName(), - "--run_display_name", c.job.DisplayName, - "--dag_execution_id", inputValue(paramParentDagID), - "--component", inputValue(paramComponent), - "--task", inputValue(paramTask), - "--task_name", inputValue(paramTaskName), - "--runtime_config", inputValue(paramRuntimeConfig), - "--iteration_index", inputValue(paramIterationIndex), - "--execution_id_path", outputPath(paramExecutionID), - "--iteration_count_path", outputPath(paramIterationCount), - "--condition_path", outputPath(paramCondition), - "--http_proxy", proxy.GetConfig().GetHttpProxy(), - "--https_proxy", proxy.GetConfig().GetHttpsProxy(), - "--no_proxy", proxy.GetConfig().GetNoProxy(), - "--ml_pipeline_server_address", config.GetMLPipelineServerConfig().Address, - "--ml_pipeline_server_port", config.GetMLPipelineServerConfig().Port, - "--mlmd_server_address", metadata.GetMetadataConfig().Address, - "--mlmd_server_port", metadata.GetMetadataConfig().Port, - } - if c.cacheDisabled { - args = append(args, "--cache_disabled") - } - if c.mlPipelineTLSEnabled { - args = append(args, "--ml_pipeline_tls_enabled") - } - if common.GetMetadataTLSEnabled() { - args = append(args, "--metadata_tls_enabled") - } - - setCABundle := false - // If CABUNDLE_SECRET_NAME or CABUNDLE_CONFIGMAP_NAME is set, add ca_cert_path arg to DAG driver. + return name, nil + } + + args := map[string]interface{}{ + "type": inputValue(paramDriverType), + "pipeline_name": c.spec.GetPipelineInfo().GetName(), + "run_id": runID(), + "run_name": runResourceName(), + "run_display_name": c.job.DisplayName, + "dag_execution_id": inputValue(paramParentDagID), + "component": inputValue(paramComponent), + "task": inputValue(paramTask), + "task_name": inputValue(paramTaskName), + "runtime_config": inputValue(paramRuntimeConfig), + "iteration_index": inputValue(paramIterationIndex), + "execution_id_path": outputPath(paramExecutionID), + "iteration_count_path": outputPath(paramIterationCount), + "condition_path": outputPath(paramCondition), + "http_proxy": proxy.GetConfig().GetHttpProxy(), + "https_proxy": proxy.GetConfig().GetHttpsProxy(), + "no_proxy": proxy.GetConfig().GetNoProxy(), + "ml_pipeline_server_address": config.GetMLPipelineServerConfig().Address, + "ml_pipeline_server_port": config.GetMLPipelineServerConfig().Port, + "mlmd_server_address": metadata.GetMetadataConfig().Address, + "mlmd_server_port": metadata.GetMetadataConfig().Port, + "cache_disabled": c.cacheDisabled, + "ml_pipeline_tls_enabled": c.mlPipelineTLSEnabled, + "metadata_tls_enabled": common.GetMetadataTLSEnabled(), + } + // If CABUNDLE_SECRET_NAME add ca_cert_path arg to DAG driver. if common.GetCaBundleSecretName() != "" || common.GetCaBundleConfigMapName() != "" { - args = append(args, "--ca_cert_path", common.CustomCaCertPath) - setCABundle = true + args["ca_cert_path"] = common.CustomCaCertPath } if value, ok := os.LookupEnv(PipelineLogLevelEnvVar); ok { - args = append(args, "--log_level", value) + args["log_level"] = value } if value, ok := os.LookupEnv(PublishLogsEnvVar); ok { - args = append(args, "--publish_logs", value) + args["publish_logs"] = value + } + + dagPlugin, err := driverPlugin(args) + if err != nil { + return "", err } template := &wfapi.Template{ @@ -630,27 +633,17 @@ func (c *workflowCompiler) addDAGDriverTemplate() string { }, Outputs: wfapi.Outputs{ Parameters: []wfapi.Parameter{ - {Name: paramExecutionID, ValueFrom: &wfapi.ValueFrom{Path: "/tmp/outputs/execution-id"}}, - {Name: paramIterationCount, ValueFrom: &wfapi.ValueFrom{Path: "/tmp/outputs/iteration-count", Default: wfapi.AnyStringPtr("0")}}, - {Name: paramCondition, ValueFrom: &wfapi.ValueFrom{Path: "/tmp/outputs/condition", Default: wfapi.AnyStringPtr("true")}}, + {Name: paramExecutionID, ValueFrom: &wfapi.ValueFrom{JSONPath: "$.execution-id"}}, + {Name: paramIterationCount, ValueFrom: &wfapi.ValueFrom{JSONPath: "$.iteration-count", Default: wfapi.AnyStringPtr("0")}}, + {Name: paramCondition, ValueFrom: &wfapi.ValueFrom{JSONPath: "$.condition", Default: wfapi.AnyStringPtr("true")}}, }, }, - Container: &k8score.Container{ - Image: c.driverImage, - Command: c.driverCommand, - Args: args, - Resources: driverResources, - Env: proxy.GetConfig().GetEnvVars(), - }, + Plugin: dagPlugin, } applySecurityContextToTemplate(template) - // If TLS is enabled (apiserver or metadata), add the custom CA bundle to the DAG driver template. - if setCABundle { - ConfigureCustomCABundle(template) - } c.templates[name] = template c.wf.Spec.Templates = append(c.wf.Spec.Templates, *template) - return name + return name, nil } func addImplicitDependencies(dagSpec *pipelinespec.DagSpec) error { diff --git a/backend/src/v2/compiler/argocompiler/plugin.go b/backend/src/v2/compiler/argocompiler/plugin.go new file mode 100644 index 00000000000..eb5f379a6dc --- /dev/null +++ b/backend/src/v2/compiler/argocompiler/plugin.go @@ -0,0 +1,39 @@ +// Copyright 2025 The Kubeflow Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package argocompiler + +import ( + "encoding/json" + "fmt" + + wfapi "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" +) + +// Create the Argo Workflow executor plugin template with parameters. +// // See https://argo-workflows.readthedocs.io/en/latest/executor_plugins/ +func driverPlugin(params map[string]interface{}) (*wfapi.Plugin, error) { + pluginConfig := map[string]interface{}{ + "driver-plugin": map[string]interface{}{ + "args": params, + }, + } + jsonConfig, err := json.Marshal(pluginConfig) + if err != nil { + return nil, fmt.Errorf("driver plugin creation error: marshaling plugin config to JSON failed: %w", err) + } + return &wfapi.Plugin{Object: wfapi.Object{ + Value: jsonConfig, + }}, nil +} diff --git a/backend/src/v2/driver/container.go b/backend/src/v2/driver/container.go index 5f10ba31500..16d84f04ee6 100644 --- a/backend/src/v2/driver/container.go +++ b/backend/src/v2/driver/container.go @@ -20,9 +20,8 @@ import ( "fmt" "strconv" - "github.com/golang/glog" - "github.com/google/uuid" "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" + "github.com/kubeflow/pipelines/backend/src/common/util" "github.com/kubeflow/pipelines/backend/src/v2/cacheutils" "github.com/kubeflow/pipelines/backend/src/v2/expression" "github.com/kubeflow/pipelines/backend/src/v2/metadata" @@ -43,14 +42,18 @@ func validateContainer(opts Options) (err error) { return validateNonRoot(opts) } -func Container(ctx context.Context, opts Options, mlmd *metadata.Client, cacheClient cacheutils.Client) (execution *Execution, err error) { +func Container(ctx context.Context, pipeline *metadata.Pipeline, opts Options, mlmd *metadata.Client, cacheClient cacheutils.Client, outputPathPrefix string) (execution *Execution, err error) { defer func() { if err != nil { err = fmt.Errorf("driver.Container(%s) failed: %w", opts.info(), err) } }() b, _ := json.Marshal(opts) - glog.V(4).Info("Container opts: ", string(b)) + log := util.GetLoggerFrom(ctx) + if log == nil { + return nil, fmt.Errorf("failed to get logger for container: %s", string(b)) + } + log.Trace("Container opts: ", string(b)) err = validateContainer(opts) if err != nil { return nil, err @@ -62,15 +65,11 @@ func Container(ctx context.Context, opts Options, mlmd *metadata.Client, cacheCl } // TODO(Bobgy): there's no need to pass any parameters, because pipeline // and pipeline run context have been created by root DAG driver. - pipeline, err := mlmd.GetPipeline(ctx, opts.PipelineName, opts.RunID, "", "", "", "") - if err != nil { - return nil, err - } dag, err := mlmd.GetDAG(ctx, opts.DAGExecutionID) if err != nil { return nil, err } - glog.Infof("parent DAG: %+v", dag.Execution) + log.Infof("parent DAG: %+v", dag.Execution) expr, err := expression.New() if err != nil { return nil, err @@ -116,7 +115,7 @@ func Container(ctx context.Context, opts Options, mlmd *metadata.Client, cacheCl pipeline.GetPipelineRoot(), opts.TaskName, opts.Component.GetOutputDefinitions(), - uuid.NewString(), + outputPathPrefix, opts.PublishLogs, ) } @@ -126,11 +125,13 @@ func Container(ctx context.Context, opts Options, mlmd *metadata.Client, cacheCl return execution, err } ecfg.TaskName = opts.TaskName + ecfg.Namespace = opts.Namespace ecfg.DisplayName = opts.Task.GetTaskInfo().GetName() ecfg.ExecutionType = metadata.ContainerExecutionTypeName ecfg.ParentDagID = dag.Execution.GetID() ecfg.IterationIndex = iterationIndex ecfg.NotTriggered = !execution.WillTrigger() + ecfg.DriverLogURI = metadata.GenerateOutputURI(pipeline.GetPipelineRoot(), []string{opts.TaskName, outputPathPrefix, "driver-logs"}, false) if isKubernetesPlatformOp { return execution, kubernetesPlatformOps(ctx, mlmd, cacheClient, execution, ecfg, &opts) @@ -181,18 +182,18 @@ func Container(ctx context.Context, opts Options, mlmd *metadata.Client, cacheCl createdExecution, err := mlmd.CreateExecution(ctx, pipeline, ecfg) if err != nil { if isAlreadyExistsErr(err) { - glog.Infof("Execution %q already exists, looking up existing execution", ecfg.Name) + log.Infof("Execution %q already exists, looking up existing execution", ecfg.Name) existing, lookupErr := mlmd.GetExecutionByTypeAndName(ctx, string(metadata.ContainerExecutionTypeName), ecfg.Name) if lookupErr != nil { return execution, fmt.Errorf("failed to lookup existing execution: %w", lookupErr) } - glog.Infof("Found existing execution: %s", existing) + log.Infof("Found existing execution: %s", existing) createdExecution = existing } else { return execution, err } } - glog.Infof("Created execution: %s", createdExecution) + log.Infof("Created execution: %s", createdExecution) execution.ID = createdExecution.GetID() if !execution.WillTrigger() { return execution, nil @@ -216,12 +217,12 @@ func Container(ctx context.Context, opts Options, mlmd *metadata.Client, cacheCl if err := mlmd.PublishExecution(ctx, createdExecution, executorOutput.GetParameterValues(), outputArtifacts, pb.Execution_CACHED); err != nil { return execution, fmt.Errorf("failed to publish cached execution: %w", err) } - glog.Infof("Use cache for task %s", opts.Task.GetTaskInfo().GetName()) + log.Infof("Use cache for task %s", opts.Task.GetTaskInfo().GetName()) *execution.Cached = true return execution, nil } } else { - glog.Info("Cache disabled globally at the server level.") + log.Info("Cache disabled globally at the server level.") } taskConfig := &TaskConfig{} diff --git a/backend/src/v2/driver/container_test.go b/backend/src/v2/driver/container_test.go index af21731480e..fd6148e5465 100644 --- a/backend/src/v2/driver/container_test.go +++ b/backend/src/v2/driver/container_test.go @@ -20,8 +20,10 @@ import ( "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" "github.com/kubeflow/pipelines/backend/src/apiserver/config/proxy" + "github.com/kubeflow/pipelines/backend/src/common/util" "github.com/kubeflow/pipelines/backend/src/v2/metadata" pb "github.com/kubeflow/pipelines/third_party/ml-metadata/go/ml_metadata" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/grpc" @@ -232,28 +234,31 @@ func TestContainer_CreateExecutionGeneralFailure(t *testing.T) { mlmdClient := metadata.NewTestClient(mockSvc) - execution, err := Container(context.Background(), Options{ - IterationIndex: -1, - PipelineName: "pipeline-1", - RunID: "run-1", - TaskName: "task-1", - Component: &pipelinespec.ComponentSpec{ - Implementation: &pipelinespec.ComponentSpec_ExecutorLabel{ExecutorLabel: "executor"}, - InputDefinitions: &pipelinespec.ComponentInputsSpec{Parameters: map[string]*pipelinespec.ComponentInputsSpec_ParameterSpec{}}, - OutputDefinitions: &pipelinespec.ComponentOutputsSpec{ - Parameters: map[string]*pipelinespec.ComponentOutputsSpec_ParameterSpec{"output": {ParameterType: pipelinespec.ParameterType_STRING}}, + pipeline := &metadata.Pipeline{} + + execution, err := Container(util.WithExistingLogger(context.Background(), logrus.New()), + pipeline, Options{ + IterationIndex: -1, + PipelineName: "pipeline-1", + RunID: "run-1", + TaskName: "task-1", + Component: &pipelinespec.ComponentSpec{ + Implementation: &pipelinespec.ComponentSpec_ExecutorLabel{ExecutorLabel: "executor"}, + InputDefinitions: &pipelinespec.ComponentInputsSpec{Parameters: map[string]*pipelinespec.ComponentInputsSpec_ParameterSpec{}}, + OutputDefinitions: &pipelinespec.ComponentOutputsSpec{ + Parameters: map[string]*pipelinespec.ComponentOutputsSpec_ParameterSpec{"output": {ParameterType: pipelinespec.ParameterType_STRING}}, + }, }, - }, - DAGExecutionID: 55, - Task: &pipelinespec.PipelineTaskSpec{ - TaskInfo: &pipelinespec.PipelineTaskInfo{Name: "task-1"}, - CachingOptions: &pipelinespec.PipelineTaskSpec_CachingOptions{EnableCache: true}, - }, - Container: &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{ - Image: "python:3.11", - Command: []string{"python", "main.py"}, - }, - }, mlmdClient, &mockCacheClient{}) + DAGExecutionID: 55, + Task: &pipelinespec.PipelineTaskSpec{ + TaskInfo: &pipelinespec.PipelineTaskInfo{Name: "task-1"}, + CachingOptions: &pipelinespec.PipelineTaskSpec_CachingOptions{EnableCache: true}, + }, + Container: &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{ + Image: "python:3.11", + Command: []string{"python", "main.py"}, + }, + }, mlmdClient, &mockCacheClient{}, "tmp") require.NotNil(t, execution) require.Error(t, err) @@ -288,28 +293,31 @@ func TestContainer_CreateExecutionSuccess(t *testing.T) { mlmdClient := metadata.NewTestClient(mockSvc) - execution, err := Container(context.Background(), Options{ - IterationIndex: -1, - PipelineName: "pipeline-1", - RunID: "run-1", - TaskName: "task-1", - Component: &pipelinespec.ComponentSpec{ - Implementation: &pipelinespec.ComponentSpec_ExecutorLabel{ExecutorLabel: "executor"}, - InputDefinitions: &pipelinespec.ComponentInputsSpec{Parameters: map[string]*pipelinespec.ComponentInputsSpec_ParameterSpec{}}, - OutputDefinitions: &pipelinespec.ComponentOutputsSpec{ - Parameters: map[string]*pipelinespec.ComponentOutputsSpec_ParameterSpec{"output": {ParameterType: pipelinespec.ParameterType_STRING}}, + pipeline := &metadata.Pipeline{} + + execution, err := Container(util.WithExistingLogger(context.Background(), logrus.New()), + pipeline, Options{ + IterationIndex: -1, + PipelineName: "pipeline-1", + RunID: "run-1", + TaskName: "task-1", + Component: &pipelinespec.ComponentSpec{ + Implementation: &pipelinespec.ComponentSpec_ExecutorLabel{ExecutorLabel: "executor"}, + InputDefinitions: &pipelinespec.ComponentInputsSpec{Parameters: map[string]*pipelinespec.ComponentInputsSpec_ParameterSpec{}}, + OutputDefinitions: &pipelinespec.ComponentOutputsSpec{ + Parameters: map[string]*pipelinespec.ComponentOutputsSpec_ParameterSpec{"output": {ParameterType: pipelinespec.ParameterType_STRING}}, + }, }, - }, - DAGExecutionID: 55, - Task: &pipelinespec.PipelineTaskSpec{ - TaskInfo: &pipelinespec.PipelineTaskInfo{Name: "task-1"}, - CachingOptions: &pipelinespec.PipelineTaskSpec_CachingOptions{EnableCache: true}, - }, - Container: &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{ - Image: "python:3.11", - Command: []string{"python", "main.py"}, - }, - }, mlmdClient, &mockCacheClient{}) + DAGExecutionID: 55, + Task: &pipelinespec.PipelineTaskSpec{ + TaskInfo: &pipelinespec.PipelineTaskInfo{Name: "task-1"}, + CachingOptions: &pipelinespec.PipelineTaskSpec_CachingOptions{EnableCache: true}, + }, + Container: &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{ + Image: "python:3.11", + Command: []string{"python", "main.py"}, + }, + }, mlmdClient, &mockCacheClient{}, "tmp") require.NotNil(t, execution) require.NoError(t, err) @@ -346,29 +354,30 @@ func TestContainer_CreateExecutionAlreadyExistsLookupReturnsNil(t *testing.T) { } mlmdClient := metadata.NewTestClient(mockSvc) - - execution, err := Container(context.Background(), Options{ - IterationIndex: -1, - PipelineName: "pipeline-1", - RunID: "run-1", - TaskName: "task-1", - Component: &pipelinespec.ComponentSpec{ - Implementation: &pipelinespec.ComponentSpec_ExecutorLabel{ExecutorLabel: "executor"}, - InputDefinitions: &pipelinespec.ComponentInputsSpec{Parameters: map[string]*pipelinespec.ComponentInputsSpec_ParameterSpec{}}, - OutputDefinitions: &pipelinespec.ComponentOutputsSpec{ - Parameters: map[string]*pipelinespec.ComponentOutputsSpec_ParameterSpec{"output": {ParameterType: pipelinespec.ParameterType_STRING}}, + pipeline := &metadata.Pipeline{} + execution, err := Container(util.WithExistingLogger(context.Background(), logrus.New()), + pipeline, Options{ + IterationIndex: -1, + PipelineName: "pipeline-1", + RunID: "run-1", + TaskName: "task-1", + Component: &pipelinespec.ComponentSpec{ + Implementation: &pipelinespec.ComponentSpec_ExecutorLabel{ExecutorLabel: "executor"}, + InputDefinitions: &pipelinespec.ComponentInputsSpec{Parameters: map[string]*pipelinespec.ComponentInputsSpec_ParameterSpec{}}, + OutputDefinitions: &pipelinespec.ComponentOutputsSpec{ + Parameters: map[string]*pipelinespec.ComponentOutputsSpec_ParameterSpec{"output": {ParameterType: pipelinespec.ParameterType_STRING}}, + }, }, - }, - DAGExecutionID: 55, - Task: &pipelinespec.PipelineTaskSpec{ - TaskInfo: &pipelinespec.PipelineTaskInfo{Name: "task-1"}, - CachingOptions: &pipelinespec.PipelineTaskSpec_CachingOptions{EnableCache: true}, - }, - Container: &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{ - Image: "python:3.11", - Command: []string{"python", "main.py"}, - }, - }, mlmdClient, &mockCacheClient{}) + DAGExecutionID: 55, + Task: &pipelinespec.PipelineTaskSpec{ + TaskInfo: &pipelinespec.PipelineTaskInfo{Name: "task-1"}, + CachingOptions: &pipelinespec.PipelineTaskSpec_CachingOptions{EnableCache: true}, + }, + Container: &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{ + Image: "python:3.11", + Command: []string{"python", "main.py"}, + }, + }, mlmdClient, &mockCacheClient{}, "tmp") require.NotNil(t, execution) require.Error(t, err) @@ -404,8 +413,9 @@ func TestContainer_CreateExecutionDoesNotExistGenericError(t *testing.T) { } mlmdClient := metadata.NewTestClient(mockSvc) + pipeline := &metadata.Pipeline{} - execution, err := Container(context.Background(), Options{ + execution, err := Container(util.WithExistingLogger(context.Background(), logrus.New()), pipeline, Options{ IterationIndex: -1, PipelineName: "pipeline-1", RunID: "run-1", @@ -426,7 +436,7 @@ func TestContainer_CreateExecutionDoesNotExistGenericError(t *testing.T) { Image: "python:3.11", Command: []string{"python", "main.py"}, }, - }, mlmdClient, &mockCacheClient{}) + }, mlmdClient, &mockCacheClient{}, "tmp") // In a successful recovery, we expect NO error to be returned from Container require.Error(t, err) diff --git a/backend/src/v2/driver/dag.go b/backend/src/v2/driver/dag.go index 362fac66f7f..39464abe2e5 100644 --- a/backend/src/v2/driver/dag.go +++ b/backend/src/v2/driver/dag.go @@ -19,8 +19,8 @@ import ( "encoding/json" "fmt" - "github.com/golang/glog" "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" + "github.com/kubeflow/pipelines/backend/src/common/util" "github.com/kubeflow/pipelines/backend/src/v2/expression" "github.com/kubeflow/pipelines/backend/src/v2/metadata" "google.golang.org/protobuf/types/known/structpb" @@ -38,14 +38,18 @@ func validateDAG(opts Options) (err error) { return validateNonRoot(opts) } -func DAG(ctx context.Context, opts Options, mlmd *metadata.Client) (execution *Execution, err error) { +func DAG(ctx context.Context, pipeline *metadata.Pipeline, opts Options, mlmd *metadata.Client) (execution *Execution, err error) { defer func() { if err != nil { err = fmt.Errorf("driver.DAG(%s) failed: %w", opts.info(), err) } }() b, _ := json.Marshal(opts) - glog.V(4).Info("DAG opts: ", string(b)) + log := util.GetLoggerFrom(ctx) + if log == nil { + return nil, fmt.Errorf("failed to get logger for DAG driver options: %s", string(b)) + } + log.Trace("DAG opts: ", string(b)) err = validateDAG(opts) if err != nil { return nil, err @@ -55,17 +59,11 @@ func DAG(ctx context.Context, opts Options, mlmd *metadata.Client) (execution *E index := opts.IterationIndex iterationIndex = &index } - // TODO(Bobgy): there's no need to pass any parameters, because pipeline - // and pipeline run context have been created by root DAG driver. - pipeline, err := mlmd.GetPipeline(ctx, opts.PipelineName, opts.RunID, "", "", "", "") - if err != nil { - return nil, err - } dag, err := mlmd.GetDAG(ctx, opts.DAGExecutionID) if err != nil { return nil, err } - glog.Infof("parent DAG: %+v", dag.Execution) + log.Infof("parent DAG: %+v", dag.Execution) expr, err := expression.New() if err != nil { return nil, err @@ -77,7 +75,7 @@ func DAG(ctx context.Context, opts Options, mlmd *metadata.Client) (execution *E executorInput := &pipelinespec.ExecutorInput{ Inputs: inputs, } - glog.Infof("executorInput value: %+v", executorInput) + log.Infof("executorInput value: %+v", executorInput) execution = &Execution{ExecutorInput: executorInput} condition := opts.Task.GetTriggerPolicy().GetCondition() if condition != "" { @@ -109,15 +107,15 @@ func DAG(ctx context.Context, opts Options, mlmd *metadata.Client) (execution *E // Handle writing output parameters to MLMD. ecfg.OutputParameters = opts.Component.GetDag().GetOutputs().GetParameters() - glog.V(4).Info("outputParameters: ", ecfg.OutputParameters) + log.Trace("outputParameters: ", ecfg.OutputParameters) // Handle writing output artifacts to MLMD. ecfg.OutputArtifacts = opts.Component.GetDag().GetOutputs().GetArtifacts() - glog.V(4).Info("outputArtifacts: ", ecfg.OutputArtifacts) + log.Trace("outputArtifacts: ", ecfg.OutputArtifacts) totalDagTasks := len(opts.Component.GetDag().GetTasks()) ecfg.TotalDagTasks = &totalDagTasks - glog.V(4).Info("totalDagTasks: ", *ecfg.TotalDagTasks) + log.Trace("totalDagTasks: ", *ecfg.TotalDagTasks) if opts.Task.GetArtifactIterator() != nil { return execution, fmt.Errorf("ArtifactIterator is not implemented") @@ -164,17 +162,17 @@ func DAG(ctx context.Context, opts Options, mlmd *metadata.Client) (execution *E execution.IterationCount = &count } - glog.V(4).Info("pipeline: ", pipeline) + log.Trace("pipeline: ", pipeline) b, _ = json.Marshal(*ecfg) - glog.V(4).Info("ecfg: ", string(b)) - glog.V(4).Infof("dag: %v", dag) + log.Trace("ecfg: ", string(b)) + log.Tracef("dag: %v", dag) // TODO(Bobgy): change execution state to pending, because this is driver, execution hasn't started. createdExecution, err := mlmd.CreateExecution(ctx, pipeline, ecfg) if err != nil { return execution, err } - glog.Infof("Created execution: %s", createdExecution) + log.Infof("Created execution: %s", createdExecution) execution.ID = createdExecution.GetID() return execution, nil } diff --git a/backend/src/v2/driver/driver.go b/backend/src/v2/driver/driver.go index 6d6188c2e61..8dfc1caeb81 100644 --- a/backend/src/v2/driver/driver.go +++ b/backend/src/v2/driver/driver.go @@ -706,7 +706,7 @@ func provisionOutputs( pipelineRoot, taskName string, outputsSpec *pipelinespec.ComponentOutputsSpec, - outputURISalt string, + prefix string, publishOutput string, ) *pipelinespec.ExecutorInput_Outputs { outputs := &pipelinespec.ExecutorInput_Outputs{ @@ -736,7 +736,7 @@ func provisionOutputs( // artifacts (dsl.get_uri) by allowing the SDK to infer the task root from // the executor output file's directory (set below) and convert it back to // a remote URI at runtime. - taskRootRemote := metadata.GenerateOutputURI(pipelineRoot, []string{taskName, outputURISalt}, false) + taskRootRemote := metadata.GenerateOutputURI(pipelineRoot, []string{taskName, prefix}, false) // Set per-artifact output URIs under the task root. for name, artifact := range artifacts { @@ -769,7 +769,6 @@ func provisionOutputs( // Fallback to legacy path if the pipeline root scheme is not recognized. outputs.OutputFile = component.OutputMetadataFilepath } - return outputs } diff --git a/backend/src/v2/driver/driver_test.go b/backend/src/v2/driver/driver_test.go index e0a1f9d0858..a7b39fe529f 100644 --- a/backend/src/v2/driver/driver_test.go +++ b/backend/src/v2/driver/driver_test.go @@ -19,6 +19,8 @@ import ( "fmt" "testing" + "github.com/kubeflow/pipelines/backend/src/common/util" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" "github.com/kubeflow/pipelines/backend/src/apiserver/config/proxy" @@ -1469,7 +1471,7 @@ func Test_initPodSpecPatch_TaskConfig_Affinity_NodeSelector_Tolerations_Passthro assert.Nil(t, err) err = extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), podSpec, opts, nil, @@ -1572,7 +1574,7 @@ func Test_initPodSpecPatch_TaskConfig_Affinity_NodeSelector_Tolerations_ApplyAnd assert.Nil(t, err) err = extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), podSpec, opts, nil, diff --git a/backend/src/v2/driver/k8s.go b/backend/src/v2/driver/k8s.go index 9716b7f11fc..82de251ead5 100644 --- a/backend/src/v2/driver/k8s.go +++ b/backend/src/v2/driver/k8s.go @@ -21,7 +21,6 @@ import ( "fmt" "time" - "github.com/golang/glog" "github.com/google/uuid" "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" "github.com/kubeflow/pipelines/backend/src/common/util" @@ -127,6 +126,10 @@ func extendPodSpecPatch( kubernetesExecutorConfig := opts.KubernetesExecutorConfig setOnTaskConfig, setOnPod := getTaskConfigOptions(opts.Component) + log := util.GetLoggerFrom(ctx) + if log == nil { + return fmt.Errorf("cannot get log from context for extendPodSpecPatch") + } // Always set setOnTaskConfig to an empty map if taskConfig is nil to avoid nil pointer dereference. if taskConfig == nil { @@ -210,7 +213,7 @@ func extendPodSpecPatch( if tolerations := kubernetesExecutorConfig.GetTolerations(); tolerations != nil { var k8sTolerations []k8score.Toleration - glog.Infof("Tolerations passed: %+v", tolerations) + log.Infof("Tolerations passed: %+v", tolerations) for _, toleration := range tolerations { if toleration != nil { @@ -244,7 +247,7 @@ func extendPodSpecPatch( } k8sTolerations = append(k8sTolerations, singleToleration) } else { - glog.V(4).Info("encountered empty tolerations struct, ignoring.") + log.Trace("encountered empty tolerations struct, ignoring.") } } else if isListToleration { listVal := resolvedParam.GetListValue() @@ -259,7 +262,7 @@ func extendPodSpecPatch( } k8sTolerations = append(k8sTolerations, k8sTolerationsList...) } else { - glog.V(4).Info("encountered empty tolerations list, ignoring.") + log.Trace("encountered empty tolerations list, ignoring.") } } else { return fmt.Errorf("encountered unexpected toleration proto value, must be either struct or list type") @@ -608,7 +611,7 @@ func extendPodSpecPatch( if nodeAffinityTerm.GetNodeAffinityJson() == nil && len(nodeAffinityTerm.GetMatchExpressions()) == 0 && len(nodeAffinityTerm.GetMatchFields()) == 0 { - glog.Warningf("NodeAffinityTerm %d is empty, skipping", i) + log.Warningf("NodeAffinityTerm %d is empty, skipping", i) continue } if nodeAffinityTerm.GetNodeAffinityJson() != nil { @@ -657,10 +660,10 @@ func extendPodSpecPatch( Weight: *nodeAffinityTerm.Weight, Preference: nodeSelectorTerm, }) - glog.V(4).Infof("Added preferred node affinity: %+v", nodeSelectorTerm) + log.Tracef("Added preferred node affinity: %+v", nodeSelectorTerm) } else { requiredTerms = append(requiredTerms, nodeSelectorTerm) - glog.V(4).Infof("Added required node affinity: %+v", nodeSelectorTerm) + log.Tracef("Added required node affinity: %+v", nodeSelectorTerm) } } @@ -729,18 +732,18 @@ func extendPodSpecPatch( isCompilerHardened := existingSecurityContext.AllowPrivilegeEscalation != nil && !*existingSecurityContext.AllowPrivilegeEscalation if userSecurityContext.RunAsUser != nil { if existingSecurityContext.RunAsUser != nil { - glog.Warningf("Ignoring user-specified runAsUser (%d): security context already set by admin (runAsUser=%d)", + log.Warningf("Ignoring user-specified runAsUser (%d): security context already set by admin (runAsUser=%d)", *userSecurityContext.RunAsUser, *existingSecurityContext.RunAsUser) } else { if isCompilerHardened && *userSecurityContext.RunAsUser == 0 { - glog.Warningf("Setting runAsUser=0 (root) on a container with hardened security context; consider using a non-root UID") + log.Warningf("Setting runAsUser=0 (root) on a container with hardened security context; consider using a non-root UID") } podSpec.Containers[0].SecurityContext.RunAsUser = userSecurityContext.RunAsUser } } if userSecurityContext.RunAsGroup != nil { if existingSecurityContext.RunAsGroup != nil { - glog.Warningf("Ignoring user-specified runAsGroup (%d): security context already set by admin (runAsGroup=%d)", + log.Warningf("Ignoring user-specified runAsGroup (%d): security context already set by admin (runAsGroup=%d)", *userSecurityContext.RunAsGroup, *existingSecurityContext.RunAsGroup) } else { podSpec.Containers[0].SecurityContext.RunAsGroup = userSecurityContext.RunAsGroup @@ -748,7 +751,7 @@ func extendPodSpecPatch( } if userSecurityContext.RunAsNonRoot != nil { if existingSecurityContext.RunAsNonRoot != nil { - glog.Warningf("Ignoring user-specified runAsNonRoot (%v): security context already set by admin (runAsNonRoot=%v)", + log.Warningf("Ignoring user-specified runAsNonRoot (%v): security context already set by admin (runAsNonRoot=%v)", *userSecurityContext.RunAsNonRoot, *existingSecurityContext.RunAsNonRoot) } else { podSpec.Containers[0].SecurityContext.RunAsNonRoot = userSecurityContext.RunAsNonRoot @@ -786,8 +789,13 @@ func createPVC( taskStartedTime := time.Now().Unix() + log := util.GetLoggerFrom(ctx) + if log == nil { + return "", nil, 0, fmt.Errorf("failed to get log from context") + } + inputs := execution.ExecutorInput.Inputs - glog.Infof("Input parameter values: %+v", inputs.ParameterValues) + log.Infof("Input parameter values: %+v", inputs.ParameterValues) // Required input: access_modes accessModeInput, ok := inputs.ParameterValues["access_modes"] @@ -879,7 +887,7 @@ func createPVC( if err != nil { return "", createdExecution, pb.Execution_FAILED, fmt.Errorf("error creating MLMD execution for createpvc: %w", err) } - glog.Infof("Created execution: %s", createdExecution) + log.Infof("Created execution: %s", createdExecution) execution.ID = createdExecution.GetID() if !execution.WillTrigger() { return "", createdExecution, pb.Execution_COMPLETE, nil @@ -930,7 +938,7 @@ func createPVC( if err != nil { return "", createdExecution, pb.Execution_FAILED, fmt.Errorf("failed to create pvc: %w", err) } - glog.Infof("Created PVC %s\n", createdPVC.ObjectMeta.Name) + log.Infof("Created PVC %s\n", createdPVC.Name) // Create a cache entry if !opts.CacheDisabled && opts.Task.GetCachingOptions().GetEnableCache() { @@ -986,9 +994,13 @@ func deletePVC( }() taskStartedTime := time.Now().Unix() + log := util.GetLoggerFrom(ctx) + if log == nil { + return nil, pb.Execution_FAILED, fmt.Errorf("no logs available for execution") + } inputs := execution.ExecutorInput.Inputs - glog.Infof("Input parameter values: %+v", inputs.ParameterValues) + log.Infof("Input parameter values: %+v", inputs.ParameterValues) // Required input: pvc_name pvcNameInput, ok := inputs.ParameterValues["pvc_name"] @@ -1018,7 +1030,7 @@ func deletePVC( if err != nil { return createdExecution, pb.Execution_FAILED, fmt.Errorf("error creating MLMD execution for createpvc: %w", err) } - glog.Infof("Created execution: %s", createdExecution) + log.Infof("Created execution: %s", createdExecution) execution.ID = createdExecution.GetID() if !execution.WillTrigger() { return createdExecution, pb.Execution_COMPLETE, nil @@ -1057,7 +1069,7 @@ func deletePVC( return createdExecution, pb.Execution_FAILED, fmt.Errorf("failed to delete pvc %s: %v", pvcName, err) } - glog.Infof("Deleted PVC %s\n", pvcName) + log.Infof("Deleted PVC %s\n", pvcName) // Create a cache entry if !opts.CacheDisabled && opts.Task.GetCachingOptions().GetEnableCache() && ecfg.CachedMLMDExecutionID != "" { @@ -1173,7 +1185,11 @@ func publishDriverExecution( if err = mlmd.PublishExecution(ctx, execution, outputParameters, outputArtifacts, status); err != nil { return fmt.Errorf("failed to publish: %w", err) } - glog.Infof("Published execution of Kubernetes platform task %s.", execution.TaskName()) + log := util.GetLoggerFrom(ctx) + if log == nil { + return fmt.Errorf("can not get log from the context ") + } + log.Infof("Published execution of Kubernetes platform task %s.", execution.TaskName()) return nil } diff --git a/backend/src/v2/driver/k8s_test.go b/backend/src/v2/driver/k8s_test.go index d1834756435..46215c30a93 100644 --- a/backend/src/v2/driver/k8s_test.go +++ b/backend/src/v2/driver/k8s_test.go @@ -5,8 +5,10 @@ import ( "testing" "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" + "github.com/kubeflow/pipelines/backend/src/common/util" "github.com/kubeflow/pipelines/backend/src/v2/metadata" "github.com/kubeflow/pipelines/kubernetes_platform/go/kubernetesplatform" + "github.com/sirupsen/logrus" "github.com/spf13/viper" "github.com/stretchr/testify/assert" "google.golang.org/protobuf/types/known/structpb" @@ -102,7 +104,7 @@ func Test_makeVolumeMountPatch(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { volumeMounts, volumes, err := makeVolumeMountPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), Options{}, tt.args.pvcMount, tt.args.dag, @@ -238,7 +240,7 @@ func Test_makePodSpecPatch_nodeSelector(t *testing.T) { taskConfig := &TaskConfig{} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -717,7 +719,7 @@ func Test_extendPodSpecPatch_Secret(t *testing.T) { taskConfig := &TaskConfig{} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), tt.podSpec, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -1214,7 +1216,7 @@ func Test_extendPodSpecPatch_ConfigMap(t *testing.T) { taskConfig := &TaskConfig{} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), tt.podSpec, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -1388,7 +1390,7 @@ func Test_extendPodSpecPatch_EmptyVolumeMount(t *testing.T) { taskConfig := &TaskConfig{} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), tt.podSpec, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -1514,7 +1516,7 @@ func Test_extendPodSpecPatch_ImagePullSecrets(t *testing.T) { }, }} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -1953,7 +1955,7 @@ func Test_extendPodSpecPatch_Tolerations(t *testing.T) { taskConfig := &TaskConfig{} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -2060,7 +2062,7 @@ func Test_extendPodSpecPatch_FieldPathAsEnv(t *testing.T) { taskConfig := &TaskConfig{} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -2133,7 +2135,7 @@ func Test_extendPodSpecPatch_ActiveDeadlineSeconds(t *testing.T) { }, }} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -2246,7 +2248,7 @@ func Test_extendPodSpecPatch_SecurityContext(t *testing.T) { }, }} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -2274,7 +2276,7 @@ func Test_extendPodSpecPatch_SecurityContext_CombinedWithOtherFeatures(t *testin }, }} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{KubernetesExecutorConfig: &kubernetesplatform.KubernetesExecutorConfig{ SecurityContext: &kubernetesplatform.SecurityContext{ @@ -2321,7 +2323,7 @@ func Test_extendPodSpecPatch_SecurityContext_AdminSetPreserved(t *testing.T) { {Name: "main"}, }} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{ DefaultRunAsUser: &adminUID, @@ -2355,7 +2357,7 @@ func Test_extendPodSpecPatch_SecurityContext_AdminDefaultsNoUserOverride(t *test {Name: "main"}, }} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{ DefaultRunAsUser: &adminUID, @@ -2383,7 +2385,7 @@ func Test_extendPodSpecPatch_SecurityContext_RootOnHardenedContainer(t *testing. }, }} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{KubernetesExecutorConfig: &kubernetesplatform.KubernetesExecutorConfig{ SecurityContext: &kubernetesplatform.SecurityContext{ @@ -2424,7 +2426,7 @@ func Test_extendPodSpecPatch_SecurityContext_AdminRunAsNonRoot(t *testing.T) { {Name: "main"}, }} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{ DefaultRunAsNonRoot: &adminRunAsNonRoot, @@ -2453,7 +2455,7 @@ func Test_extendPodSpecPatch_SecurityContext_AdminRunAsNonRootNoUserOverride(t * {Name: "main"}, }} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{ DefaultRunAsNonRoot: &adminRunAsNonRoot, @@ -2475,7 +2477,7 @@ func Test_extendPodSpecPatch_SecurityContext_UserRunAsNonRootNoAdmin(t *testing. {Name: "main"}, }} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{ KubernetesExecutorConfig: &kubernetesplatform.KubernetesExecutorConfig{ @@ -2567,7 +2569,7 @@ func Test_extendPodSpecPatch_ImagePullPolicy(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), tt.podSpec, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -2765,7 +2767,7 @@ func Test_extendPodSpecPatch_GenericEphemeralVolume(t *testing.T) { taskConfig := &TaskConfig{} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), tt.podSpec, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -3069,7 +3071,7 @@ func Test_extendPodSpecPatch_NodeAffinity(t *testing.T) { taskConfig := &TaskConfig{} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -3154,7 +3156,7 @@ func Test_extendPodSpecPatch_TaskConfig_CapturesAndApplies(t *testing.T) { taskCfg := &TaskConfig{} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), podSpec, Options{KubernetesExecutorConfig: cfg, Component: comp}, nil, @@ -3305,7 +3307,7 @@ func Test_extendPodSpecPatch_PvcMounts_Passthrough_NotAppliedToPod(t *testing.T) } taskCfg := &TaskConfig{} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), podSpec, Options{KubernetesExecutorConfig: cfg, Component: comp}, nil, @@ -3339,7 +3341,7 @@ func Test_extendPodSpecPatch_PvcMounts_Passthrough_AppliedToPod(t *testing.T) { } taskCfg := &TaskConfig{} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), podSpec, Options{KubernetesExecutorConfig: cfg, Component: comp}, nil, diff --git a/backend/src/v2/driver/root_dag.go b/backend/src/v2/driver/root_dag.go index f35f4b9a309..1fdeff5af49 100644 --- a/backend/src/v2/driver/root_dag.go +++ b/backend/src/v2/driver/root_dag.go @@ -19,7 +19,6 @@ import ( "encoding/json" "fmt" - "github.com/golang/glog" "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" "github.com/kubeflow/pipelines/backend/src/common/util" "github.com/kubeflow/pipelines/backend/src/v2/config" @@ -81,50 +80,54 @@ func validateRootDAG(opts Options) (err error) { return nil } -func RootDAG(ctx context.Context, opts Options, mlmd metadata.ClientInterface) (execution *Execution, err error) { +func RootDAG(ctx context.Context, opts Options, mlmd metadata.ClientInterface) (execution *Execution, pipeline *metadata.Pipeline, err error) { defer func() { if err != nil { err = fmt.Errorf("driver.RootDAG(%s) failed: %w", opts.info(), err) } }() b, _ := json.Marshal(opts) - glog.V(4).Info("RootDAG opts: ", string(b)) + log := util.GetLoggerFrom(ctx) + if log == nil { + return nil, nil, fmt.Errorf("driver.RootDAG(%s) failed: invalid log configuration", opts.info()) + } + log.Trace("RootDAG opts: ", string(b)) err = validateRootDAG(opts) if err != nil { - return nil, err + return nil, nil, err } // TODO(v2): in pipeline spec, rename GCS output directory to pipeline root. pipelineRoot := opts.RuntimeConfig.GetGcsOutputDirectory() k8sClient, err := buildK8sClient() if err != nil { - return nil, fmt.Errorf("failed to initialize kubernetes client set: %w", err) + return nil, nil, fmt.Errorf("failed to initialize kubernetes client set: %w", err) } cfg, err := config.FromConfigMap(ctx, k8sClient, opts.Namespace) if err != nil { - return nil, err + return nil, nil, err } storeSessionInfo := objectstore.SessionInfo{} if pipelineRoot != "" { - glog.Infof("PipelineRoot=%q", pipelineRoot) + log.Infof("PipelineRoot=%q", pipelineRoot) } else { pipelineRoot = cfg.DefaultPipelineRoot() - glog.Infof("PipelineRoot=%q from default config", pipelineRoot) + log.Infof("PipelineRoot=%q from default config", pipelineRoot) } storeSessionInfo, err = cfg.GetStoreSessionInfo(pipelineRoot) if err != nil { - return nil, err + return nil, nil, err } storeSessionInfoJSON, err := json.Marshal(storeSessionInfo) if err != nil { - return nil, err + return nil, nil, err } storeSessionInfoStr := string(storeSessionInfoJSON) // TODO(Bobgy): fill in run resource. - pipeline, err := mlmd.GetPipeline(ctx, opts.PipelineName, opts.RunID, opts.Namespace, "run-resource", pipelineRoot, storeSessionInfoStr) + pipeline, err = mlmd.GetPipeline(ctx, opts.PipelineName, opts.RunID, opts.Namespace, "run-resource", pipelineRoot, storeSessionInfoStr) if err != nil { - return nil, err + return nil, nil, err } executorInput := &pipelinespec.ExecutorInput{ @@ -135,36 +138,35 @@ func RootDAG(ctx context.Context, opts Options, mlmd metadata.ClientInterface) ( // TODO(Bobgy): validate executorInput matches component spec types ecfg, err := metadata.GenerateExecutionConfig(executorInput) if err != nil { - return nil, err + return nil, pipeline, err } ecfg.ExecutionType = metadata.DagExecutionTypeName ecfg.Name = fmt.Sprintf("run/%s", opts.RunID) exec, err := mlmd.CreateExecution(ctx, pipeline, ecfg) - glog.Infof("Creating execution: %s", exec.String()) + log.Infof("Creating execution: %s", exec.String()) if err != nil { // When Argo retries the ROOT_DAG driver, the execution was already created on the first attempt. // The deterministic name "run/" causes a duplicate key error. // Handle it here by looking up the existing record instead of failing. if isAlreadyExistsErr(err) { - glog.Infof("Execution %q already exists, looking up existing execution", ecfg.Name) + log.Infof("Execution %q already exists, looking up existing execution", ecfg.Name) existing, lookupErr := mlmd.GetExecutionByTypeAndName(ctx, string(metadata.DagExecutionTypeName), ecfg.Name) if lookupErr != nil { - return nil, fmt.Errorf("failed to lookup existing execution: %w", lookupErr) + return nil, pipeline, fmt.Errorf("failed to lookup existing execution: %w", lookupErr) } if existing == nil { - return nil, fmt.Errorf("execution already exists but lookup returned nil: %w", err) + return nil, pipeline, fmt.Errorf("execution already exists but lookup returned nil: %w", err) } - glog.Infof("Found existing execution: %s", existing) - return &Execution{ID: existing.GetID()}, nil + log.Infof("Found existing execution: %s", existing) + return &Execution{ID: existing.GetID()}, pipeline, nil } - return nil, err + return nil, pipeline, err } - glog.Infof("Created execution: %s", exec) - + log.Infof("Created execution: %s", exec) // No need to return ExecutorInput, because tasks in the DAG will resolve // needed info from MLMD. - return &Execution{ID: exec.GetID()}, nil + return &Execution{ID: exec.GetID()}, pipeline, nil } diff --git a/backend/src/v2/driver/root_dag_test.go b/backend/src/v2/driver/root_dag_test.go index 1438cc6ffee..9eadba8ed69 100644 --- a/backend/src/v2/driver/root_dag_test.go +++ b/backend/src/v2/driver/root_dag_test.go @@ -20,8 +20,10 @@ import ( "github.com/go-openapi/errors" "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" + "github.com/kubeflow/pipelines/backend/src/common/util" "github.com/kubeflow/pipelines/backend/src/v2/metadata" pb "github.com/kubeflow/pipelines/third_party/ml-metadata/go/ml_metadata" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" @@ -194,28 +196,29 @@ func TestRootDAG_CreateExecutionAlreadyExistsReturnsExistingID(t *testing.T) { return fake.NewClientset(mockConfigMap), nil } - execution, err := RootDAG(context.Background(), Options{ - PipelineName: "pipeline-1", - RunID: "run-1", - Component: &pipelinespec.ComponentSpec{}, - RuntimeConfig: &pipelinespec.PipelineJob_RuntimeConfig{}, - Namespace: "kubeflow", - IterationIndex: -1, - }, &metadata.FakeClient{ - CreateExecutionFunc: func(ctx context.Context, pipeline *metadata.Pipeline, config *metadata.ExecutionConfig) (*metadata.Execution, error) { - return nil, errors.New(6, "AlreadyExists") - }, - GetPipelineFunc: func(ctx context.Context, pipelineName, runID, namespace, runResource, pipelineRoot string, storeSessionInfo string) (*metadata.Pipeline, error) { - return &metadata.Pipeline{}, nil - }, - GetExecutionByTypeAndNameFunc: func(ctx context.Context, typeName, name string) (*metadata.Execution, error) { - return &metadata.Execution{ - Execution: &pb.Execution{ - Id: new(int64(1234)), - }, - }, nil - }, - }) + execution, _, err := RootDAG(util.WithExistingLogger(context.Background(), logrus.New()), + Options{ + PipelineName: "pipeline-1", + RunID: "run-1", + Component: &pipelinespec.ComponentSpec{}, + RuntimeConfig: &pipelinespec.PipelineJob_RuntimeConfig{}, + Namespace: "kubeflow", + IterationIndex: -1, + }, &metadata.FakeClient{ + CreateExecutionFunc: func(ctx context.Context, pipeline *metadata.Pipeline, config *metadata.ExecutionConfig) (*metadata.Execution, error) { + return nil, errors.New(6, "AlreadyExists") + }, + GetPipelineFunc: func(ctx context.Context, pipelineName, runID, namespace, runResource, pipelineRoot string, storeSessionInfo string) (*metadata.Pipeline, error) { + return &metadata.Pipeline{}, nil + }, + GetExecutionByTypeAndNameFunc: func(ctx context.Context, typeName, name string) (*metadata.Execution, error) { + return &metadata.Execution{ + Execution: &pb.Execution{ + Id: new(int64(1234)), + }, + }, nil + }, + }) require.NoError(t, err) require.NotNil(t, execution) @@ -241,25 +244,26 @@ func TestRootDAG_CreateExecutionAlreadyExistsLookupFailure(t *testing.T) { return fake.NewClientset(mockConfigMap), nil } - execution, err := RootDAG(context.Background(), Options{ - PipelineName: "pipeline-1", - RunID: "run-1", - Component: &pipelinespec.ComponentSpec{}, - RuntimeConfig: &pipelinespec.PipelineJob_RuntimeConfig{}, - Namespace: "kubeflow", - IterationIndex: -1, - }, &metadata.FakeClient{ - CreateExecutionFunc: func(ctx context.Context, pipeline *metadata.Pipeline, config *metadata.ExecutionConfig) (*metadata.Execution, error) { - return nil, errors.New(6, "AlreadyExists") - }, - GetPipelineFunc: func(ctx context.Context, pipelineName, runID, namespace, runResource, pipelineRoot string, storeSessionInfo string) (*metadata.Pipeline, error) { - return &metadata.Pipeline{}, nil - }, - // Error on lookup - GetExecutionByTypeAndNameFunc: func(ctx context.Context, typeName, name string) (*metadata.Execution, error) { - return nil, errors.New(0, "Failed to connect (test)") - }, - }) + execution, _, err := RootDAG(util.WithExistingLogger(context.Background(), logrus.New()), + Options{ + PipelineName: "pipeline-1", + RunID: "run-1", + Component: &pipelinespec.ComponentSpec{}, + RuntimeConfig: &pipelinespec.PipelineJob_RuntimeConfig{}, + Namespace: "kubeflow", + IterationIndex: -1, + }, &metadata.FakeClient{ + CreateExecutionFunc: func(ctx context.Context, pipeline *metadata.Pipeline, config *metadata.ExecutionConfig) (*metadata.Execution, error) { + return nil, errors.New(6, "AlreadyExists") + }, + GetPipelineFunc: func(ctx context.Context, pipelineName, runID, namespace, runResource, pipelineRoot string, storeSessionInfo string) (*metadata.Pipeline, error) { + return &metadata.Pipeline{}, nil + }, + // Error on lookup + GetExecutionByTypeAndNameFunc: func(ctx context.Context, typeName, name string) (*metadata.Execution, error) { + return nil, errors.New(0, "Failed to connect (test)") + }, + }) require.Nil(t, execution) require.Error(t, err) assert.Contains(t, err.Error(), "Failed to connect (test)") @@ -285,25 +289,26 @@ func TestRootDAG_CreateExecutionAlreadyExistsLookupFailure_NilExistingRecord(t * return fake.NewClientset(mockConfigMap), nil } - execution, err := RootDAG(context.Background(), Options{ - PipelineName: "pipeline-1", - RunID: "run-1", - Component: &pipelinespec.ComponentSpec{}, - RuntimeConfig: &pipelinespec.PipelineJob_RuntimeConfig{}, - Namespace: "kubeflow", - IterationIndex: -1, - }, &metadata.FakeClient{ - CreateExecutionFunc: func(ctx context.Context, pipeline *metadata.Pipeline, config *metadata.ExecutionConfig) (*metadata.Execution, error) { - return nil, errors.New(6, "AlreadyExists") - }, - GetPipelineFunc: func(ctx context.Context, pipelineName, runID, namespace, runResource, pipelineRoot string, storeSessionInfo string) (*metadata.Pipeline, error) { - return &metadata.Pipeline{}, nil - }, - // Lookup succeeds but record is nil - GetExecutionByTypeAndNameFunc: func(ctx context.Context, typeName, name string) (*metadata.Execution, error) { - return nil, nil - }, - }) + execution, _, err := RootDAG(util.WithExistingLogger(context.Background(), logrus.New()), + Options{ + PipelineName: "pipeline-1", + RunID: "run-1", + Component: &pipelinespec.ComponentSpec{}, + RuntimeConfig: &pipelinespec.PipelineJob_RuntimeConfig{}, + Namespace: "kubeflow", + IterationIndex: -1, + }, &metadata.FakeClient{ + CreateExecutionFunc: func(ctx context.Context, pipeline *metadata.Pipeline, config *metadata.ExecutionConfig) (*metadata.Execution, error) { + return nil, errors.New(6, "AlreadyExists") + }, + GetPipelineFunc: func(ctx context.Context, pipelineName, runID, namespace, runResource, pipelineRoot string, storeSessionInfo string) (*metadata.Pipeline, error) { + return &metadata.Pipeline{}, nil + }, + // Lookup succeeds but record is nil + GetExecutionByTypeAndNameFunc: func(ctx context.Context, typeName, name string) (*metadata.Execution, error) { + return nil, nil + }, + }) require.Nil(t, execution) require.Error(t, err) assert.Contains(t, err.Error(), "execution already exists but lookup returned nil") diff --git a/backend/src/v2/metadata/client.go b/backend/src/v2/metadata/client.go index 201126668c6..cf1a3e03981 100644 --- a/backend/src/v2/metadata/client.go +++ b/backend/src/v2/metadata/client.go @@ -207,6 +207,7 @@ type ExecutionConfig struct { OutputArtifacts map[string]*pipelinespec.DagOutputsSpec_DagOutputArtifactSpec InputArtifactIDs map[string][]int64 IterationIndex *int // Index of the iteration. + DriverLogURI string // ContainerExecution custom properties Image, CachedMLMDExecutionID, FingerPrint string @@ -603,6 +604,7 @@ const ( keyOutputs = "outputs" keyParameterProducerTask = "parameter_producer_task" keyOutputArtifacts = "output_artifacts" + keyDriverLogURI = "driver_logs_uri" keyArtifactProducerTask = "artifact_producer_task" keyParentDagID = "parent_dag_id" // Parent DAG Execution ID. keyIterationIndex = "iteration_index" @@ -660,6 +662,10 @@ func (c *Client) CreateExecution(ctx context.Context, pipeline *Pipeline, config e.CustomProperties[keyCacheFingerPrint] = StringValue(config.FingerPrint) } } + if config.DriverLogURI != "" { + e.CustomProperties[keyDriverLogURI] = StringValue(config.DriverLogURI) + e.CustomProperties[keyStoreSessionInfo] = StringValue(pipeline.GetStoreSessionInfo()) + } if config.InputParameters != nil { e.CustomProperties[keyInputs] = &pb.Value{Value: &pb.Value_StructValue{ StructValue: &structpb.Struct{ diff --git a/backend/test/compiler/utils/workflow_utils.go b/backend/test/compiler/utils/workflow_utils.go index 1cb7a8ac5f6..ebae3d2c31a 100644 --- a/backend/test/compiler/utils/workflow_utils.go +++ b/backend/test/compiler/utils/workflow_utils.go @@ -36,6 +36,8 @@ import ( v1 "k8s.io/api/core/v1" ) +type driverPlugin map[string]map[string]map[string]interface{} + // LoadPipelineSpecsFromIR - Unmarshall Pipeline Spec IR into a tuple of (pipelinespec.PipelineJob, pipelinespec.SinglePlatformSpec) func LoadPipelineSpecsFromIR(pipelineIRFilePath string, cacheDisabled bool, defaultWorkspace *v1.PersistentVolumeClaimSpec) (*pipelinespec.PipelineJob, *pipelinespec.SinglePlatformSpec) { pipelineSpecsFromFile := testutil.ParseFileToSpecs(pipelineIRFilePath, cacheDisabled, defaultWorkspace) @@ -89,6 +91,32 @@ func CreateCompiledWorkflowFile(compiledWorflow *v1alpha1.Workflow, compiledWork return testutil.CreateFile(compiledWorkflowFilePath, [][]byte{fileContents}) } +// ConfigurePluginSettings - Add/Remove cache_disabled args in the driver-plugin +func ConfigurePluginSettings(workflow *v1alpha1.Workflow, remove bool) *v1alpha1.Workflow { + configuredWorkflow := workflow.DeepCopy() + for i, template := range configuredWorkflow.Spec.Templates { + if template.Plugin != nil { + var pluginMap driverPlugin + if err := json.Unmarshal(template.Plugin.Value, &pluginMap); err == nil { + if driverPlugin, ok := pluginMap["driver-plugin"]; ok { + if args, ok := driverPlugin["args"]; ok { + if remove { + args["cache_disabled"] = false + } else { + args["cache_disabled"] = true + } + } + } + jsonPlugin, err := json.Marshal(pluginMap) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to marshal plugin map") + configuredWorkflow.Spec.Templates[i].Plugin.Value = jsonPlugin + } + } + } + + return configuredWorkflow +} + // ConfigureCacheSettings - Add/Remove cache_disabled args in the workflow func ConfigureCacheSettings(workflow *v1alpha1.Workflow, remove bool) *v1alpha1.Workflow { cacheDisabledArg := "--cache_disabled" @@ -138,5 +166,5 @@ func ConfigureCacheSettings(workflow *v1alpha1.Workflow, remove bool) *v1alpha1. } } } - return configuredWorkflow + return ConfigurePluginSettings(configuredWorkflow, remove) } diff --git a/backend/test/end2end/e2e_suite_test.go b/backend/test/end2end/e2e_suite_test.go index a765a3dab9d..9a8d0aff951 100644 --- a/backend/test/end2end/e2e_suite_test.go +++ b/backend/test/end2end/e2e_suite_test.go @@ -135,7 +135,6 @@ var _ = BeforeSuite(func() { }) var _ = BeforeEach(func() { - // Create Experiment so that we can use it to associate pipeline runs with experimentName := fmt.Sprintf("E2EExperiment-%s", strconv.FormatInt(time.Now().UnixNano(), 10)) experiment := testutil.CreateExperiment(experimentClient, experimentName, testutil.GetNamespace()) diff --git a/backend/test/end2end/utils/e2e_utils.go b/backend/test/end2end/utils/e2e_utils.go index 96d5cf78c73..31e8e14cb48 100644 --- a/backend/test/end2end/utils/e2e_utils.go +++ b/backend/test/end2end/utils/e2e_utils.go @@ -5,6 +5,7 @@ import ( "fmt" "maps" "sort" + "strings" "time" runparams "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/run_client/run_service" @@ -74,7 +75,8 @@ func ValidateComponentStatuses(runClient *apiserver.RunClient, k8Client *kuberne logger.Log("Updated pipeline run details") expectedTaskDetails := GetTasksFromWorkflow(compiledWorkflow) if *updatedRun.State == run_model.V2beta1RuntimeStateRUNNING { - logger.Log("Pipeline run did not finish, checking workflow controller logs") + logger.Log("Pipeline run did not finish") + logger.Log("Checking workflow controller logs") podLog := testutil.ReadContainerLogs(k8Client, *config.Namespace, "workflow-controller", nil, &testContext.TestStartTimeUTC, config.PodLogLimit) logger.Log("Attaching Workflow Controller logs to the report") ginkgo.AddReportEntry("Workflow Controller Logs", podLog) @@ -121,11 +123,24 @@ func CapturePodLogsForUnsuccessfulTasks(k8Client *kubernetes.Clientset, testCont } case run_model.V2beta1RuntimeStateFAILED: { + agentLogsNotYetAcquired := true logger.Log("%s - Task %s for Run %s did not complete successfully", *task.State, task.DisplayName, task.RunID) for _, childTask := range task.ChildTasks { podName := childTask.PodName + isDriver := strings.Contains(task.DisplayName, "-driver") if podName != "" { - logger.Log("Capturing pod logs for task %s, with pod name %s", task.DisplayName, podName) + if isDriver { + if agentLogsNotYetAcquired { + logger.Log("Capturing pod logs for task executor agent, with pod name %s", podName) + podName += "-agent" + agentLogsNotYetAcquired = false + } else { + logger.Log("Logs from agent pod was already captured") + continue + } + } else { + logger.Log("Capturing pod logs for task %s, with pod name %s", task.DisplayName, podName) + } podLog := testutil.ReadPodLogs(k8Client, *config.Namespace, podName, nil, &testContext.TestStartTimeUTC, config.PodLogLimit) logger.Log("Pod logs captured for task %s in pod %s", task.DisplayName, podName) logger.Log("Attaching pod logs to the report") diff --git a/backend/test/testutil/kubernetes_utils.go b/backend/test/testutil/kubernetes_utils.go index 6cdc218ed70..99173851d56 100644 --- a/backend/test/testutil/kubernetes_utils.go +++ b/backend/test/testutil/kubernetes_utils.go @@ -25,7 +25,6 @@ import ( "github.com/kubeflow/pipelines/backend/src/common/util" "github.com/kubeflow/pipelines/backend/test/logger" - "github.com/onsi/gomega" authenticationv1 "k8s.io/api/authentication/v1" v1 "k8s.io/api/core/v1" @@ -76,10 +75,6 @@ func ReadPodLogs(client *kubernetes.Clientset, namespace string, podName string, podLogOptions.Container = container.Name podLogsRequest := client.CoreV1().Pods(namespace).GetLogs(podFromPodName.Name, podLogOptions) podLogs, err := podLogsRequest.Stream(context.Background()) // Pass a context for cancellation - if err != nil { - logger.Log("Failed to stream pod logs for container '%s' due to %v", container.Name, err) - continue - } if podLogs == nil { logger.Log("Pod log stream is nil for container '%s'", container.Name) continue diff --git a/frontend/src/components/tabs/RuntimeNodeDetailsV2.tsx b/frontend/src/components/tabs/RuntimeNodeDetailsV2.tsx index 03f9ab108e1..bf08cd80fa2 100644 --- a/frontend/src/components/tabs/RuntimeNodeDetailsV2.tsx +++ b/frontend/src/components/tabs/RuntimeNodeDetailsV2.tsx @@ -62,6 +62,9 @@ import { getComponentSpec } from 'src/lib/v2/NodeUtils'; export const LOGS_DETAILS = 'logs_details'; export const LOGS_BANNER_MESSAGE = 'logs_banner_message'; export const LOGS_BANNER_ADDITIONAL_INFO = 'logs_banner_additional_info'; +export const SYS_LOGS_DETAILS = 'sys_log_details'; +export const SYS_LOGS_BANNER_MESSAGE = 'sys_logs_banner_message'; +export const SYS_LOGS_BANNER_ADDITIONAL_INFO = 'sys_logs_banner_additional_info'; export const K8S_PLATFORM_KEY = 'kubernetes'; const NODE_INFO_UNKNOWN = ( @@ -170,19 +173,33 @@ function TaskNodeDetail({ enabled: !!execution, }); + const { data: driverLogsInfo } = useQuery({ + queryKey: queryKeys.driverLogs(execution?.getId(), namespace), + queryFn: async (): Promise> => { + if (!execution) { + throw new Error('No execution is found.'); + } + return getDriverLogsInfo(execution, namespace); + }, + enabled: !!execution, + }); + const logsDetails = logsInfo?.get(LOGS_DETAILS); const logsBannerMessage = logsInfo?.get(LOGS_BANNER_MESSAGE) || (logsQueryFailed ? 'Failed to retrieve pod logs.' : undefined); const logsBannerAdditionalInfo = logsInfo?.get(LOGS_BANNER_ADDITIONAL_INFO) || logsQueryError?.message; + const sysLogDetails = driverLogsInfo?.get(SYS_LOGS_DETAILS); + const sysLogsBannerMessage = driverLogsInfo?.get(SYS_LOGS_BANNER_MESSAGE); + const sysLogsBannerAdditionalInfo = driverLogsInfo?.get(SYS_LOGS_BANNER_ADDITIONAL_INFO); const [selectedTab, setSelectedTab] = useState(0); return (
setSelectedTab(tab)} /> @@ -219,6 +236,24 @@ function TaskNodeDetail({ )}
)} + {/* System Logs tab */} + {selectedTab === 3 && ( +
+ {sysLogsBannerMessage && ( + + + + )} + {!sysLogsBannerMessage && ( +
+ +
+ )} +
+ )} ); @@ -306,6 +341,51 @@ function getNodeVolumeMounts( return volumeMounts; } +async function getDriverLogsInfo( + execution: Execution, + namespace?: string, +): Promise> { + const logsInfo = new Map(); + let podNameSpace = ''; + let logsDetails = ''; + let logsBannerMessage = ''; + let logsBannerAdditionalInfo = ''; + const customPropertiesMap = execution.getCustomPropertiesMap(); + + if (execution) { + podNameSpace = customPropertiesMap.get('namespace')?.getStringValue() || ''; + } + + try { + const driverLogUri = customPropertiesMap.get('driver_logs_uri')?.getStringValue(); + const storeSessionInfo = customPropertiesMap.get('store_session_info')?.getStringValue(); + + if (driverLogUri && storeSessionInfo) { + const storagePath = WorkflowParser.parseStoragePath(driverLogUri); + const providerInfo = storeSessionInfo; + const artifactNamespace = namespace || podNameSpace; + + logsDetails = await Apis.readFile({ + path: storagePath, + providerInfo: providerInfo, + namespace: artifactNamespace, + }); + logsInfo.set(SYS_LOGS_DETAILS, logsDetails); + return logsInfo; + } + } catch (artifactErr) { + let errMsg = await errorToMessage(artifactErr); + logsBannerMessage = 'Failed to retrieve system logs.'; + logsInfo.set(SYS_LOGS_BANNER_MESSAGE, logsBannerMessage); + logsBannerAdditionalInfo = 'Error response: ' + errMsg; + logsInfo.set(SYS_LOGS_BANNER_ADDITIONAL_INFO, logsBannerAdditionalInfo); + + console.error('Failed to retrieve driver-logs artifact:', artifactErr); + } + + return logsInfo; +} + async function getLogsInfo( execution: Execution, runId?: string, diff --git a/frontend/src/hooks/queryKeys.ts b/frontend/src/hooks/queryKeys.ts index cc70d74effd..f229524fba2 100644 --- a/frontend/src/hooks/queryKeys.ts +++ b/frontend/src/hooks/queryKeys.ts @@ -65,6 +65,9 @@ export const queryKeys = { executionLogs: (executionId: number | undefined, namespace: string | undefined) => ['execution_logs', { executionId, namespace }] as const, + driverLogs: (executionId: number | undefined, namespace: string | undefined) => + ['driver_logs', { executionId, namespace }] as const, + contextByExecution: (executionId: number, executionState: number) => ['context_by_execution', { id: executionId, state: executionState }] as const, diff --git a/manifests/kustomize/base/installs/multi-user/kustomization.yaml b/manifests/kustomize/base/installs/multi-user/kustomization.yaml index dab31b61c87..8d7767f28b4 100644 --- a/manifests/kustomize/base/installs/multi-user/kustomization.yaml +++ b/manifests/kustomize/base/installs/multi-user/kustomization.yaml @@ -6,6 +6,7 @@ resources: - ../../cache-deployer/cluster-scoped - ../generic - view-edit-cluster-roles.yaml +- ml-pipeline-driver-agent-executor-plugin-cluster-role.yaml - api-service - pipelines-ui - pipelines-profile-controller diff --git a/manifests/kustomize/base/installs/multi-user/ml-pipeline-driver-agent-executor-plugin-cluster-role.yaml b/manifests/kustomize/base/installs/multi-user/ml-pipeline-driver-agent-executor-plugin-cluster-role.yaml new file mode 100644 index 00000000000..beb30b13fae --- /dev/null +++ b/manifests/kustomize/base/installs/multi-user/ml-pipeline-driver-agent-executor-plugin-cluster-role.yaml @@ -0,0 +1,32 @@ +# Since the driver is now extracted into a separate agent pod and injected as a sidecar container, +# it requires its own dedicated ServiceAccount with appropriate RBAC permissions. +# This is necessary to allow the driver to interact with the Kubernetes API +# (e.g., watching pods, creating PVCs, accessing ConfigMaps and specific Secrets) +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ml-pipeline-driver-agent-executor-plugin-cluster-role +rules: + # Execution is usually published by the launcher, but for Kubernetes-specific + # operations we bypass the default flow. In these cases, the driver uses the + # Kubernetes API to list pods during publishing. see publishDriverExecution method + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] + # the driver may need permissions to create and delete PVCs + # see test_data/sdk_compiled_pipelines/valid/pipeline_with_volume.py for example + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["create", "get", "list", "delete"] + # The root DAG driver reads the runtime configuration from the kfp-launcher cm + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["kfp-launcher"] + verbs: ["get", "list", "watch"] + # The driver now uploads its logs to artifact storage so they are visible in the UI + # for improved troubleshooting. To do this, it needs access to the + # `mlpipeline-minio-artifact` Secret containing storage credentials. + - apiGroups: [""] + resources: ["secrets"] + resourceNames: ["mlpipeline-minio-artifact"] + verbs: ["get"] \ No newline at end of file diff --git a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/decorator-controller.yaml b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/decorator-controller.yaml index 318e5fd2fae..e4275da1168 100644 --- a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/decorator-controller.yaml +++ b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/decorator-controller.yaml @@ -27,6 +27,18 @@ spec: resource: services updateStrategy: method: InPlace + - apiVersion: v1 + resource: serviceaccounts + updateStrategy: + method: InPlace + - apiVersion: rbac.authorization.k8s.io/v1 + resource: roles + updateStrategy: + method: InPlace + - apiVersion: rbac.authorization.k8s.io/v1 + resource: rolebindings + updateStrategy: + method: InPlace - apiVersion: networking.istio.io/v1alpha3 resource: destinationrules updateStrategy: diff --git a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py index 12fa8bd1926..bca8ba4c888 100644 --- a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py +++ b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py @@ -173,6 +173,58 @@ def upsert_lifecycle_policy(self, bucket_name, artifact_retention_days): else: print(f"ERROR: Failed to configure lifecycle policy: {exception}") + def executor_plugin_resources(self, namespace): + print('Creating executor-plugin resources for namespace:', namespace) + # Argo Workflow Executor Plugin Necessary Resources + agent_service_account_name = 'ml-pipeline-driver-agent-executor-plugin' + agent_secret_name = f'{agent_service_account_name}.service-account-token' + return [ + { + 'apiVersion': 'v1', + 'kind': 'ServiceAccount', + 'metadata': { + 'name': agent_service_account_name, + 'namespace': namespace, + 'labels': { + 'application-crd-id': 'kubeflow-pipelines' + } + }, + 'secrets': [{ + 'name': agent_secret_name + }] + }, + { + 'apiVersion': 'rbac.authorization.k8s.io/v1', + 'kind': 'RoleBinding', + 'metadata': { + 'name': 'ml-pipeline-driver-agent-executor-plugin-binding', + 'namespace': namespace, + }, + 'subjects': [{ + 'kind': 'ServiceAccount', + 'name': agent_service_account_name, + 'namespace': namespace, + }], + 'roleRef': { + 'kind': 'ClusterRole', + 'name': 'ml-pipeline-driver-agent-executor-plugin-cluster-role', + 'apiGroup': 'rbac.authorization.k8s.io', + } + }, + { + 'apiVersion': 'v1', + 'kind': 'Secret', + 'metadata': { + 'name': agent_secret_name, + 'namespace': namespace, + 'annotations': { + 'kubernetes.io/service-account.name': agent_service_account_name, + }, + }, + 'type': 'kubernetes.io/service-account-token', + }, + ] + def sync(self, parent, attachments): # parent is a namespace @@ -186,12 +238,18 @@ def sync(self, parent, attachments): # Compute status based on observed state. desired_status = { - "kubeflow-pipelines-ready": - len(attachments["Secret.v1"]) == 1 and - len(attachments["ConfigMap.v1"]) == 3 and - len(attachments["Deployment.apps/v1"]) == (1 if artifacts_proxy_enabled.lower() == "true" else 0) and - len(attachments["Service.v1"]) == (1 if artifacts_proxy_enabled.lower() == "true" else 0) and - "True" or "False" + 'kubeflow-pipelines-ready': + len(attachments['Secret.v1']) == 3 and + len(attachments['ConfigMap.v1']) == 3 and + len(attachments['Deployment.apps/v1']) == + (1 if artifacts_proxy_enabled.lower() == 'true' else 0) and + len(attachments['Service.v1']) == + (1 if artifacts_proxy_enabled.lower() == 'true' else 0) and + len(attachments['ServiceAccount.v1']) == 2 and + len(attachments['Role.rbac.authorization.k8s.io/v1']) == 4 + and + len(attachments['RoleBinding.rbac.authorization.k8s.io/v1']) + == 4 and 'True' or 'False' } # Generate the desired attachment object(s). @@ -253,7 +311,9 @@ def sync(self, parent, attachments): }, ] - # Add artifact fetcher related resources if enabled + executor_plugin_resources = self.executor_plugin_resources(namespace) + desired_resources.extend(executor_plugin_resources) + if artifacts_proxy_enabled.lower() == "true": desired_resources.extend([ { @@ -369,6 +429,20 @@ def sync(self, parent, attachments): } }, ]) + desired_resources.extend([ + { + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": "default-editor.service-account-token", + "namespace": namespace, + "annotations": { + "kubernetes.io/service-account.name": "default-editor" + } + }, + "type": "kubernetes.io/service-account-token" + }, + ]) print('Received request:\n', json.dumps(parent, sort_keys=True)) print('Desired resources except secrets:\n', json.dumps(desired_resources, sort_keys=True)) diff --git a/manifests/kustomize/base/installs/multi-user/view-edit-cluster-roles.yaml b/manifests/kustomize/base/installs/multi-user/view-edit-cluster-roles.yaml index 8c86b7971ae..838963a783b 100644 --- a/manifests/kustomize/base/installs/multi-user/view-edit-cluster-roles.yaml +++ b/manifests/kustomize/base/installs/multi-user/view-edit-cluster-roles.yaml @@ -98,7 +98,8 @@ rules: - workfloweventbindings - workflowtemplates - workflowtaskresults - + - workflowtasksets/status + - workflowtasksets --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/manifests/kustomize/base/pipeline/kustomization.yaml b/manifests/kustomize/base/pipeline/kustomization.yaml index 1ddeeb298e5..c819d38cdfc 100644 --- a/manifests/kustomize/base/pipeline/kustomization.yaml +++ b/manifests/kustomize/base/pipeline/kustomization.yaml @@ -34,6 +34,7 @@ resources: - container-builder-sa.yaml - viewer-sa.yaml - kfp-launcher-configmap.yaml + - ml-pipeline-driver-plugin-cm.yaml - public_configmap_role.yaml - public_configmap_role_binding.yaml configMapGenerator: diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml new file mode 100644 index 00000000000..edc3f415971 --- /dev/null +++ b/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml @@ -0,0 +1,45 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ml-pipeline-driver-agent + labels: + workflows.argoproj.io/configmap-type: ExecutorPlugin +data: + sidecar.automountServiceAccountToken: "true" + sidecar.container: | + name: driver-plugin + image: ghcr.io/kubeflow/kfp-driver:dummy + imagePullPolicy: IfNotPresent + env: + - name: LOG_ACCESS_KEY + valueFrom: + secretKeyRef: + name: mlpipeline-minio-artifact + key: accesskey + - name: LOG_SECRET_KEY + valueFrom: + secretKeyRef: + name: mlpipeline-minio-artifact + key: secretkey + ports: + - containerPort: 8080 + resources: + requests: + cpu: "0.1" + memory: "64Mi" + limits: + cpu: "0.5" + memory: "0.5Gi" + securityContext: + runAsNonRoot: true + runAsUser: 65534 + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: var-run-argo + mountPath: /kfp/log + readOnly: false diff --git a/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml b/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml index eba0ee9f2d6..311ad4229fb 100644 --- a/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml +++ b/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml @@ -36,12 +36,16 @@ rules: - argoproj.io resources: - workflows + - workflowtaskresults + - workflowtasksets + - "workflowtasksets/status" verbs: - get - - list + - create + - patch - watch + - list - update - - patch - apiGroups: - "" resources: @@ -85,3 +89,62 @@ rules: verbs: - create - patch + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ml-pipeline-driver-agent-executor-plugin + labels: + application-crd-id: kubeflow-pipelines +secrets: + - name: ml-pipeline-driver-agent-executor-plugin.service-account-token +--- +apiVersion: v1 +kind: Secret +metadata: + name: ml-pipeline-driver-agent-executor-plugin.service-account-token + annotations: + kubernetes.io/service-account.name: ml-pipeline-driver-agent-executor-plugin +type: kubernetes.io/service-account-token +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ml-pipeline-executor-plugin-role +rules: + # Execution is usually published by the launcher, but for Kubernetes-specific + # operations we bypass the default flow. In these cases, the driver uses the + # Kubernetes API to list pods during publishing. see publishDriverExecution method + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] + # the driver may need permissions to create and delete PVCs + # see test_data/sdk_compiled_pipelines/valid/pipeline_with_volume.py for example + - apiGroups: [ "" ] + resources: [ "persistentvolumeclaims" ] + verbs: ["create", "get", "list", "delete"] + # The root DAG driver reads the runtime configuration from the kfp-launcher cm + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["kfp-launcher"] + verbs: ["get", "list", "watch"] + # The driver now uploads its logs to artifact storage so they are visible in the UI + # for improved troubleshooting. To do this, it needs access to the + # `mlpipeline-minio-artifact` Secret containing storage credentials. + - apiGroups: [""] + resources: ["secrets"] + resourceNames: ["mlpipeline-minio-artifact"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ml-pipeline-driver-agent-executor-plugin-binding +subjects: + - kind: ServiceAccount + name: ml-pipeline-driver-agent-executor-plugin +roleRef: + kind: Role + name: ml-pipeline-executor-plugin-role + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/manifests/kustomize/base/pipeline/pipeline-runner-sa.yaml b/manifests/kustomize/base/pipeline/pipeline-runner-sa.yaml index 8cb2c669fb2..e1f42088371 100644 --- a/manifests/kustomize/base/pipeline/pipeline-runner-sa.yaml +++ b/manifests/kustomize/base/pipeline/pipeline-runner-sa.yaml @@ -2,3 +2,12 @@ apiVersion: v1 kind: ServiceAccount metadata: name: pipeline-runner + +--- +apiVersion: v1 +kind: Secret +metadata: + name: pipeline-runner.service-account-token + annotations: + kubernetes.io/service-account.name: pipeline-runner +type: kubernetes.io/service-account-token diff --git a/manifests/kustomize/env/cert-manager/base-tls-certs/kfp-api-cert.yaml b/manifests/kustomize/env/cert-manager/base-tls-certs/kfp-api-cert.yaml index 912eafb9028..2ce0cfe3d55 100644 --- a/manifests/kustomize/env/cert-manager/base-tls-certs/kfp-api-cert.yaml +++ b/manifests/kustomize/env/cert-manager/base-tls-certs/kfp-api-cert.yaml @@ -25,4 +25,6 @@ spec: issuerRef: kind: Issuer name: kfp-api-tls-selfsigned-issuer - secretName: kfp-api-tls-cert + # Unfortunately, Argo Workflows (for security reasons) can mount certificates + # into the agent only from a Secret with a predefined name `argo-workflows-agent-ca-certificates` + secretName: argo-workflows-agent-ca-certificates \ No newline at end of file diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/kustomization.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/kustomization.yaml index 6a89ad57f0e..ca012a212ee 100644 --- a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/kustomization.yaml +++ b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/kustomization.yaml @@ -42,6 +42,10 @@ patches: target: kind: Deployment name: ml-pipeline-persistenceagent + - path: patches/ml-pipeline-driver-plugin-cm.yaml + target: + kind: ConfigMap + name: ml-pipeline-driver-agent - path: patches/ml-pipeline-ui-deployment.yaml target: kind: Deployment diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-envoy-deployment.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-envoy-deployment.yaml index 0fa72587d37..c6a99880058 100644 --- a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-envoy-deployment.yaml +++ b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-envoy-deployment.yaml @@ -20,7 +20,7 @@ spec: volumes: - name: tls-certs secret: - secretName: kfp-api-tls-cert + secretName: argo-workflows-agent-ca-certificates - name: envoy-config-tls-enabled configMap: name: envoy-config diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-grpc-deployment.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-grpc-deployment.yaml index 4bc193483d7..e6af2e5f545 100644 --- a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-grpc-deployment.yaml +++ b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-grpc-deployment.yaml @@ -10,7 +10,7 @@ spec: volumes: - name: tls-certs secret: - secretName: kfp-api-tls-cert + secretName: argo-workflows-agent-ca-certificates - name: grpc-tls-config emptyDir: { } - name: mysql-secret diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-writer-deployment.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-writer-deployment.yaml index 1c651cc3b4b..f5f72c4b111 100644 --- a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-writer-deployment.yaml +++ b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-writer-deployment.yaml @@ -19,4 +19,4 @@ spec: volumes: - name: tls-certs secret: - secretName: kfp-api-tls-cert + secretName: argo-workflows-agent-ca-certificates diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-apiserver-deployment.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-apiserver-deployment.yaml index fef772368b5..59d268fe922 100644 --- a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-apiserver-deployment.yaml +++ b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-apiserver-deployment.yaml @@ -17,7 +17,7 @@ spec: - "--tlsCertKeyPath=/etc/pki/tls/certs/tls.key" env: - name: CABUNDLE_SECRET_NAME - value: "kfp-api-tls-cert" + value: "argo-workflows-agent-ca-certificates" - name: METADATA_TLS_ENABLED value: "true" - name: ML_PIPELINE_SERVICE_HOST @@ -53,4 +53,4 @@ spec: volumes: - name: tls-certs secret: - secretName: kfp-api-tls-cert + secretName: argo-workflows-agent-ca-certificates diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-driver-plugin-cm.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-driver-plugin-cm.yaml new file mode 100644 index 00000000000..3e0cd462d34 --- /dev/null +++ b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-driver-plugin-cm.yaml @@ -0,0 +1,45 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ml-pipeline-driver-agent +data: + sidecar.automountServiceAccountToken: "true" + sidecar.container: | + name: driver-plugin + image: ghcr.io/kubeflow/kfp-driver:dummy + env: + - name: LOG_ACCESS_KEY + valueFrom: + secretKeyRef: + name: mlpipeline-minio-artifact + key: accesskey + - name: LOG_SECRET_KEY + valueFrom: + secretKeyRef: + name: mlpipeline-minio-artifact + key: secretkey + ports: + - containerPort: 8080 + resources: + requests: + cpu: "0.1" + memory: "64Mi" + limits: + cpu: "0.5" + memory: "0.5Gi" + securityContext: + runAsNonRoot: true + runAsUser: 65534 + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: argo-workflows-agent-ca-certificates + mountPath: /kfp/certs + readOnly: true + - name: var-run-argo + mountPath: /kfp/log + readOnly: false diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-persistenceagent-deployment.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-persistenceagent-deployment.yaml index fccb9d2fedc..73f64d3ac06 100644 --- a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-persistenceagent-deployment.yaml +++ b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-persistenceagent-deployment.yaml @@ -35,4 +35,4 @@ spec: volumes: - name: tls-certs secret: - secretName: kfp-api-tls-cert + secretName: argo-workflows-agent-ca-certificates diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-scheduledworkflow-deployment.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-scheduledworkflow-deployment.yaml index 301ca3bb816..7c08cb7d4af 100644 --- a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-scheduledworkflow-deployment.yaml +++ b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-scheduledworkflow-deployment.yaml @@ -22,4 +22,4 @@ spec: volumes: - name: tls-certs secret: - secretName: kfp-api-tls-cert + secretName: argo-workflows-agent-ca-certificates diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-ui-deployment.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-ui-deployment.yaml index cf7775542b2..aed2004e31c 100644 --- a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-ui-deployment.yaml +++ b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-ui-deployment.yaml @@ -30,4 +30,4 @@ spec: volumes: - name: tls-certs secret: - secretName: kfp-api-tls-cert + secretName: argo-workflows-agent-ca-certificates diff --git a/manifests/kustomize/env/dev/driver-plugin-cm-path.yaml b/manifests/kustomize/env/dev/driver-plugin-cm-path.yaml new file mode 100644 index 00000000000..a30de3e9059 --- /dev/null +++ b/manifests/kustomize/env/dev/driver-plugin-cm-path.yaml @@ -0,0 +1,41 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ml-pipeline-driver-agent +data: + sidecar.container: | + name: driver-plugin + image: ghcr.io/kubeflow/kfp-driver:master + env: + - name: LOG_ACCESS_KEY + valueFrom: + secretKeyRef: + name: mlpipeline-minio-artifact + key: accesskey + - name: LOG_SECRET_KEY + valueFrom: + secretKeyRef: + name: mlpipeline-minio-artifact + key: secretkey + ports: + - containerPort: 8080 + resources: + requests: + cpu: "0.1" + memory: "64Mi" + limits: + cpu: "0.5" + memory: 0.5Gi + securityContext: + runAsNonRoot: true + runAsUser: 65534 + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: var-run-argo + mountPath: /kfp/log + readOnly: false diff --git a/manifests/kustomize/env/dev/kustomization.yaml b/manifests/kustomize/env/dev/kustomization.yaml index dd64fb6eedd..6f326244e12 100644 --- a/manifests/kustomize/env/dev/kustomization.yaml +++ b/manifests/kustomize/env/dev/kustomization.yaml @@ -11,6 +11,8 @@ resources: images: - name: ghcr.io/kubeflow/kfp-api-server newTag: master +- name: ghcr.io/kubeflow/kfp-driver + newTag: master - name: ghcr.io/kubeflow/kfp-frontend newTag: master - name: ghcr.io/kubeflow/kfp-persistence-agent @@ -34,6 +36,7 @@ images: patches: - path: api-server-patch.yaml + - path: driver-plugin-cm-path.yaml # !!! If you want to customize the namespace, # please refer sample/cluster-scoped-resources to update the namespace for cluster-scoped-resources namespace: kubeflow diff --git a/manifests/kustomize/third-party/application/application-controller-deployment.yaml b/manifests/kustomize/third-party/application/application-controller-deployment.yaml index 1f1c589aaef..63497ef2c32 100644 --- a/manifests/kustomize/third-party/application/application-controller-deployment.yaml +++ b/manifests/kustomize/third-party/application/application-controller-deployment.yaml @@ -34,5 +34,5 @@ spec: memory: 30Mi requests: cpu: 100m - memory: 20Mi + memory: 64Mi serviceAccountName: application diff --git a/manifests/kustomize/third-party/argo/base/kustomization.yaml b/manifests/kustomize/third-party/argo/base/kustomization.yaml index aee1edf4532..a7593f9dfc1 100644 --- a/manifests/kustomize/third-party/argo/base/kustomization.yaml +++ b/manifests/kustomize/third-party/argo/base/kustomization.yaml @@ -3,6 +3,7 @@ kind: Kustomization resources: - https://github.com/argoproj/argo-workflows/manifests/base/workflow-controller?ref=v3.7.3 +- workflow-controller-argo-taskset-clusterrole.yaml patches: - path: workflow-controller-deployment-patch.yaml diff --git a/manifests/kustomize/third-party/argo/base/workflow-controller-argo-taskset-clusterrole.yaml b/manifests/kustomize/third-party/argo/base/workflow-controller-argo-taskset-clusterrole.yaml new file mode 100644 index 00000000000..4a660f8cff2 --- /dev/null +++ b/manifests/kustomize/third-party/argo/base/workflow-controller-argo-taskset-clusterrole.yaml @@ -0,0 +1,27 @@ +# Argo Workflow and the Agent communicate via the new WorkflowTaskSet CR, +# so RBAC permissions are required for this resource. +# Details: https://argo-workflows.readthedocs.io/en/latest/http-template/#argo-agent-rbac +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: argo-workflowtaskset-status +rules: + - apiGroups: ["argoproj.io"] + resources: ["workflowtasksets"] + verbs: ["get","list","watch","update","patch","delete","create"] + - apiGroups: [ "argoproj.io" ] + resources: [ "workflowtasksets/status" ] + verbs: ["get", "update", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: argo-workflowtaskset-status-binding +subjects: + - kind: ServiceAccount + name: argo + namespace: kubeflow +roleRef: + kind: ClusterRole + name: argo-workflowtaskset-status + apiGroup: rbac.authorization.k8s.io diff --git a/manifests/kustomize/third-party/argo/base/workflow-controller-deployment-patch.yaml b/manifests/kustomize/third-party/argo/base/workflow-controller-deployment-patch.yaml index abeec455e3d..d13da9e498f 100644 --- a/manifests/kustomize/third-party/argo/base/workflow-controller-deployment-patch.yaml +++ b/manifests/kustomize/third-party/argo/base/workflow-controller-deployment-patch.yaml @@ -17,6 +17,12 @@ spec: - workflow-controller-configmap - --executor-image - quay.io/argoproj/argoexec:v3.7.3 + env: + # https://argo-workflows.readthedocs.io/en/latest/environment-variables/ + - name: ARGO_EXECUTOR_PLUGINS + value: "true" + - name: DEFAULT_REQUEUE_TIME + value: "6s" securityContext: readOnlyRootFilesystem: true runAsNonRoot: true diff --git a/manifests/kustomize/third-party/argo/installs/namespace/kustomization.yaml b/manifests/kustomize/third-party/argo/installs/namespace/kustomization.yaml index 989bd6c39de..8e40852c1d8 100644 --- a/manifests/kustomize/third-party/argo/installs/namespace/kustomization.yaml +++ b/manifests/kustomize/third-party/argo/installs/namespace/kustomization.yaml @@ -17,4 +17,4 @@ patches: group: apps kind: Deployment name: workflow-controller - version: v1 + version: v1 \ No newline at end of file diff --git a/manifests/kustomize/third-party/metacontroller/base/cluster-role.yaml b/manifests/kustomize/third-party/metacontroller/base/cluster-role.yaml index 1dd5e4e1ba8..1e17a673379 100644 --- a/manifests/kustomize/third-party/metacontroller/base/cluster-role.yaml +++ b/manifests/kustomize/third-party/metacontroller/base/cluster-role.yaml @@ -18,9 +18,21 @@ rules: - apiGroups: [""] resources: ["services"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +- apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["roles", "rolebindings"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - apiGroups: ["networking.istio.io"] resources: ["destinationrules"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["create", "get", "list", "delete"] - apiGroups: ["security.istio.io"] resources: ["authorizationpolicies"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] diff --git a/test_data/compiled-workflows/add_numbers.yaml b/test_data/compiled-workflows/add_numbers.yaml index de6ffdefd65..b869c6857c1 100644 --- a/test_data/compiled-workflows/add_numbers.yaml +++ b/test_data/compiled-workflows/add_numbers.yaml @@ -24,6 +24,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -32,81 +33,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - add-numbers - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -124,16 +51,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: add-numbers + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -278,70 +233,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - add-numbers - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -362,15 +254,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: add-numbers + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/arguments_parameters.yaml b/test_data/compiled-workflows/arguments_parameters.yaml index 6e33503119a..0996be86811 100644 --- a/test_data/compiled-workflows/arguments_parameters.yaml +++ b/test_data/compiled-workflows/arguments_parameters.yaml @@ -15,6 +15,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -23,81 +24,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -115,16 +42,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: echo + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -269,70 +224,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -353,15 +245,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: echo + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/artifact_cache.yaml b/test_data/compiled-workflows/artifact_cache.yaml index 401b4d7d9d8..5cb39414dc4 100644 --- a/test_data/compiled-workflows/artifact_cache.yaml +++ b/test_data/compiled-workflows/artifact_cache.yaml @@ -43,6 +43,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -51,81 +52,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - artifact-cache-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -143,16 +70,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: artifact-cache-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -297,70 +252,7 @@ spec: metadata: {} name: comp-core outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - artifact-cache-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -381,15 +273,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: artifact-cache-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/artifact_crust.yaml b/test_data/compiled-workflows/artifact_crust.yaml index 1e55ef9d2a9..3c9ee00b662 100644 --- a/test_data/compiled-workflows/artifact_crust.yaml +++ b/test_data/compiled-workflows/artifact_crust.yaml @@ -43,6 +43,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -51,81 +52,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - artifact-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -143,16 +70,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: artifact-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -297,70 +252,7 @@ spec: metadata: {} name: comp-core outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - artifact-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -381,15 +273,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: artifact-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/artifacts_complex.yaml b/test_data/compiled-workflows/artifacts_complex.yaml index 23f3c90176a..f45fc18adf9 100644 --- a/test_data/compiled-workflows/artifacts_complex.yaml +++ b/test_data/compiled-workflows/artifacts_complex.yaml @@ -69,6 +69,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -77,81 +78,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -169,16 +96,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: math-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -404,70 +359,7 @@ spec: metadata: {} name: comp-condition-5 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -488,15 +380,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: math-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/artifacts_simple.yaml b/test_data/compiled-workflows/artifacts_simple.yaml index 980c19b8ec0..feaf7bdb842 100644 --- a/test_data/compiled-workflows/artifacts_simple.yaml +++ b/test_data/compiled-workflows/artifacts_simple.yaml @@ -54,6 +54,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -62,81 +63,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -154,16 +81,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: math-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -308,70 +263,7 @@ spec: metadata: {} name: comp-for-loop-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -392,15 +284,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: math-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/collected_artifacts.yaml b/test_data/compiled-workflows/collected_artifacts.yaml index 92272c9cd13..a17e56ce43b 100644 --- a/test_data/compiled-workflows/collected_artifacts.yaml +++ b/test_data/compiled-workflows/collected_artifacts.yaml @@ -135,6 +135,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -143,81 +144,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - collected-artifact-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -235,16 +162,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: collected-artifact-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -389,70 +344,7 @@ spec: metadata: {} name: comp-single-node-dag outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - collected-artifact-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -473,15 +365,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: collected-artifact-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/collected_parameters.yaml b/test_data/compiled-workflows/collected_parameters.yaml index 0dcc4c50b89..31ac6915d3c 100644 --- a/test_data/compiled-workflows/collected_parameters.yaml +++ b/test_data/compiled-workflows/collected_parameters.yaml @@ -70,6 +70,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -78,81 +79,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - collected-param-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -170,16 +97,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: collected-param-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -349,70 +304,7 @@ spec: metadata: {} name: comp-for-loop-1 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - collected-param-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -433,15 +325,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: collected-param-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/component_with_env_variable.yaml b/test_data/compiled-workflows/component_with_env_variable.yaml index ef146aab162..6cd46cca042 100644 --- a/test_data/compiled-workflows/component_with_env_variable.yaml +++ b/test_data/compiled-workflows/component_with_env_variable.yaml @@ -25,6 +25,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -33,81 +34,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-env-variable - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -125,16 +52,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-env-variable + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -279,70 +234,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-env-variable - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -363,15 +255,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-env-variable + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/component_with_metadata_fields.yaml b/test_data/compiled-workflows/component_with_metadata_fields.yaml index 8a379d4acaa..3a47e2aa20c 100644 --- a/test_data/compiled-workflows/component_with_metadata_fields.yaml +++ b/test_data/compiled-workflows/component_with_metadata_fields.yaml @@ -39,6 +39,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -47,81 +48,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - dataset-joiner - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -139,16 +66,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: dataset-joiner + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -293,70 +248,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - dataset-joiner - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -377,15 +269,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: dataset-joiner + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/component_with_optional_inputs.yaml b/test_data/compiled-workflows/component_with_optional_inputs.yaml index f8e8d63a33b..c8d7e8fd540 100644 --- a/test_data/compiled-workflows/component_with_optional_inputs.yaml +++ b/test_data/compiled-workflows/component_with_optional_inputs.yaml @@ -28,6 +28,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -36,81 +37,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - v2-component-optional-input - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -128,16 +55,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: v2-component-optional-input + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -282,70 +237,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - v2-component-optional-input - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -366,15 +258,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: v2-component-optional-input + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/component_with_pip_index_urls.yaml b/test_data/compiled-workflows/component_with_pip_index_urls.yaml index cb5e4cf00a7..6ce2c40c39b 100644 --- a/test_data/compiled-workflows/component_with_pip_index_urls.yaml +++ b/test_data/compiled-workflows/component_with_pip_index_urls.yaml @@ -26,6 +26,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -34,81 +35,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - v2-component-pip-index-urls - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -126,16 +53,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: v2-component-pip-index-urls + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -280,70 +235,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - v2-component-pip-index-urls - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -364,15 +256,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: v2-component-pip-index-urls + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/component_with_pip_install.yaml b/test_data/compiled-workflows/component_with_pip_install.yaml index bc771212680..4bcd5c7af2c 100644 --- a/test_data/compiled-workflows/component_with_pip_install.yaml +++ b/test_data/compiled-workflows/component_with_pip_install.yaml @@ -26,6 +26,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -34,81 +35,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - component-with-pip-install - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -126,16 +53,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: component-with-pip-install + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -280,70 +235,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - component-with-pip-install - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -364,15 +256,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: component-with-pip-install + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/component_with_pip_install_in_venv.yaml b/test_data/compiled-workflows/component_with_pip_install_in_venv.yaml index 4f6c6ef817b..56aa8fa755b 100644 --- a/test_data/compiled-workflows/component_with_pip_install_in_venv.yaml +++ b/test_data/compiled-workflows/component_with_pip_install_in_venv.yaml @@ -27,6 +27,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -35,81 +36,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - component-with-pip-install - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -127,16 +54,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: component-with-pip-install + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -281,70 +236,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - component-with-pip-install - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -365,15 +257,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: component-with-pip-install + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/components_with_optional_artifacts.yaml b/test_data/compiled-workflows/components_with_optional_artifacts.yaml index 304042d1a4c..7b38f02e47e 100644 --- a/test_data/compiled-workflows/components_with_optional_artifacts.yaml +++ b/test_data/compiled-workflows/components_with_optional_artifacts.yaml @@ -40,6 +40,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -48,81 +49,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - optional-artifact-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -140,16 +67,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: optional-artifact-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -395,70 +350,7 @@ spec: runAsNonRoot: true seccompProfile: type: RuntimeDefault - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - optional-artifact-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -479,15 +371,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: optional-artifact-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/concat_message.yaml b/test_data/compiled-workflows/concat_message.yaml index 83347d58281..cbb6d947025 100644 --- a/test_data/compiled-workflows/concat_message.yaml +++ b/test_data/compiled-workflows/concat_message.yaml @@ -25,6 +25,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -33,81 +34,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - concat-message - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -125,16 +52,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: concat-message + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -279,70 +234,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - concat-message - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -363,15 +255,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: concat-message + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/conditional_producer_and_consumers.yaml b/test_data/compiled-workflows/conditional_producer_and_consumers.yaml index eeef66b184b..f35923dad16 100644 --- a/test_data/compiled-workflows/conditional_producer_and_consumers.yaml +++ b/test_data/compiled-workflows/conditional_producer_and_consumers.yaml @@ -46,6 +46,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -54,81 +55,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -146,16 +73,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: math-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -332,70 +287,7 @@ spec: metadata: {} name: comp-condition-3 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -416,15 +308,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: math-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/container_component_with_no_inputs.yaml b/test_data/compiled-workflows/container_component_with_no_inputs.yaml index 9c2081fe84e..ac4f49fb9d1 100644 --- a/test_data/compiled-workflows/container_component_with_no_inputs.yaml +++ b/test_data/compiled-workflows/container_component_with_no_inputs.yaml @@ -15,6 +15,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -23,81 +24,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - v2-container-component-no-input - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -115,16 +42,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: v2-container-component-no-input + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -269,70 +224,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - v2-container-component-no-input - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -353,15 +245,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: v2-container-component-no-input + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/container_io.yaml b/test_data/compiled-workflows/container_io.yaml index d22e741c5c0..6873daabb70 100644 --- a/test_data/compiled-workflows/container_io.yaml +++ b/test_data/compiled-workflows/container_io.yaml @@ -15,6 +15,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -23,81 +24,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - container-io - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -115,16 +42,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: container-io + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -269,70 +224,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - container-io - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -353,15 +245,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: container-io + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/container_no_input.yaml b/test_data/compiled-workflows/container_no_input.yaml index 923a04b6e85..8f309206f57 100644 --- a/test_data/compiled-workflows/container_no_input.yaml +++ b/test_data/compiled-workflows/container_no_input.yaml @@ -15,6 +15,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -23,81 +24,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - container-no-input - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -115,16 +42,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: container-no-input + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -269,70 +224,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - container-no-input - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -353,15 +245,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: container-no-input + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/container_with_artifact_output.yaml b/test_data/compiled-workflows/container_with_artifact_output.yaml index 4f858708083..a64a9dde943 100644 --- a/test_data/compiled-workflows/container_with_artifact_output.yaml +++ b/test_data/compiled-workflows/container_with_artifact_output.yaml @@ -15,6 +15,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -23,81 +24,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - container-with-artifact-output - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -115,16 +42,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: container-with-artifact-output + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -269,70 +224,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - container-with-artifact-output - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -353,15 +245,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: container-with-artifact-output + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/container_with_concat_placeholder.yaml b/test_data/compiled-workflows/container_with_concat_placeholder.yaml index 89f38c22455..b8396f1d927 100644 --- a/test_data/compiled-workflows/container_with_concat_placeholder.yaml +++ b/test_data/compiled-workflows/container_with_concat_placeholder.yaml @@ -16,6 +16,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -24,81 +25,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - container-with-concat-placeholder - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -116,16 +43,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: container-with-concat-placeholder + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -270,70 +225,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - container-with-concat-placeholder - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -354,15 +246,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: container-with-concat-placeholder + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/container_with_if_placeholder.yaml b/test_data/compiled-workflows/container_with_if_placeholder.yaml index 0df3f24826e..ac4c7432981 100644 --- a/test_data/compiled-workflows/container_with_if_placeholder.yaml +++ b/test_data/compiled-workflows/container_with_if_placeholder.yaml @@ -18,6 +18,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -26,81 +27,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - container-with-if-placeholder - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -118,16 +45,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: container-with-if-placeholder + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -272,70 +227,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - container-with-if-placeholder - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -356,15 +248,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: container-with-if-placeholder + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/container_with_placeholder_in_fstring.yaml b/test_data/compiled-workflows/container_with_placeholder_in_fstring.yaml index 996046e0c58..ecb7f2a8fd2 100644 --- a/test_data/compiled-workflows/container_with_placeholder_in_fstring.yaml +++ b/test_data/compiled-workflows/container_with_placeholder_in_fstring.yaml @@ -15,6 +15,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -23,81 +24,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - container-with-placeholder-in-fstring - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -115,16 +42,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: container-with-placeholder-in-fstring + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -269,70 +224,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - container-with-placeholder-in-fstring - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -353,15 +245,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: container-with-placeholder-in-fstring + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/containerized_python_component.yaml b/test_data/compiled-workflows/containerized_python_component.yaml index c4f3550b4ac..55dce3c9938 100644 --- a/test_data/compiled-workflows/containerized_python_component.yaml +++ b/test_data/compiled-workflows/containerized_python_component.yaml @@ -15,6 +15,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -23,81 +24,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - concat-message - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -115,16 +42,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: concat-message + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -269,70 +224,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - concat-message - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -353,15 +245,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: concat-message + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/create_pod_metadata_complex.yaml b/test_data/compiled-workflows/create_pod_metadata_complex.yaml index cbac80c4550..d64af5d1e3c 100644 --- a/test_data/compiled-workflows/create_pod_metadata_complex.yaml +++ b/test_data/compiled-workflows/create_pod_metadata_complex.yaml @@ -68,6 +68,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -76,81 +77,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-pod-metadata - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -168,16 +95,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-pod-metadata + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -658,70 +613,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-pod-metadata - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -742,15 +634,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-pod-metadata + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/cross_loop_after_topology.yaml b/test_data/compiled-workflows/cross_loop_after_topology.yaml index a53b89b94a9..08bc62d7823 100644 --- a/test_data/compiled-workflows/cross_loop_after_topology.yaml +++ b/test_data/compiled-workflows/cross_loop_after_topology.yaml @@ -45,6 +45,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -53,81 +54,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - my-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -145,16 +72,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: my-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -331,70 +286,7 @@ spec: metadata: {} name: comp-for-loop-14 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - my-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -415,15 +307,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: my-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/dict_input.yaml b/test_data/compiled-workflows/dict_input.yaml index 06d27d21937..be075fa848a 100644 --- a/test_data/compiled-workflows/dict_input.yaml +++ b/test_data/compiled-workflows/dict_input.yaml @@ -24,6 +24,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -32,81 +33,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - dict-input - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -124,16 +51,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: dict-input + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -278,70 +233,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - dict-input - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -362,15 +254,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: dict-input + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/embedded_artifact.yaml b/test_data/compiled-workflows/embedded_artifact.yaml index 5275df4d526..82e99cd75bd 100644 --- a/test_data/compiled-workflows/embedded_artifact.yaml +++ b/test_data/compiled-workflows/embedded_artifact.yaml @@ -65,6 +65,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -73,81 +74,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - nb-simple - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -165,16 +92,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: nb-simple + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -343,70 +298,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - nb-simple - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -427,15 +319,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: nb-simple + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/env-var.yaml b/test_data/compiled-workflows/env-var.yaml index 0177ac3b3ed..164cd38ea7e 100644 --- a/test_data/compiled-workflows/env-var.yaml +++ b/test_data/compiled-workflows/env-var.yaml @@ -26,6 +26,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -34,81 +35,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - test-env-exists - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -126,16 +53,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: test-env-exists + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -280,70 +235,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - test-env-exists - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -364,15 +256,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: test-env-exists + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/fail_v2.yaml b/test_data/compiled-workflows/fail_v2.yaml index 49cfe93539c..e1a94c2a6a1 100644 --- a/test_data/compiled-workflows/fail_v2.yaml +++ b/test_data/compiled-workflows/fail_v2.yaml @@ -24,6 +24,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -32,81 +33,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - fail-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -124,16 +51,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: fail-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -278,70 +233,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - fail-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -362,15 +254,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: fail-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/flip_coin.yaml b/test_data/compiled-workflows/flip_coin.yaml index be8efc3a924..84a0a5cf165 100644 --- a/test_data/compiled-workflows/flip_coin.yaml +++ b/test_data/compiled-workflows/flip_coin.yaml @@ -81,6 +81,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -89,81 +90,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - conditional-execution-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -181,16 +108,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: conditional-execution-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -371,70 +326,7 @@ spec: metadata: {} name: comp-condition-3 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - conditional-execution-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -455,15 +347,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: conditional-execution-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/hello_world.yaml b/test_data/compiled-workflows/hello_world.yaml index 497c09c20b9..22b4d17abec 100644 --- a/test_data/compiled-workflows/hello_world.yaml +++ b/test_data/compiled-workflows/hello_world.yaml @@ -15,6 +15,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -23,81 +24,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -115,16 +42,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: echo + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -269,70 +224,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -353,15 +245,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: echo + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/identity.yaml b/test_data/compiled-workflows/identity.yaml index e35b003f95a..f8a62c5f391 100644 --- a/test_data/compiled-workflows/identity.yaml +++ b/test_data/compiled-workflows/identity.yaml @@ -24,6 +24,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -32,81 +33,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - identity - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -124,16 +51,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: identity + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -278,70 +233,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - identity - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -362,15 +254,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: identity + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/if_elif_else_complex.yaml b/test_data/compiled-workflows/if_elif_else_complex.yaml index 541abc3533f..78b418f0936 100644 --- a/test_data/compiled-workflows/if_elif_else_complex.yaml +++ b/test_data/compiled-workflows/if_elif_else_complex.yaml @@ -115,6 +115,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -123,81 +124,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - lucky-number-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -215,16 +142,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: lucky-number-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -370,70 +325,7 @@ spec: metadata: {} name: comp-condition-3 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - lucky-number-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -454,15 +346,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: lucky-number-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/if_elif_else_with_oneof_parameters.yaml b/test_data/compiled-workflows/if_elif_else_with_oneof_parameters.yaml index eae6957a429..49b54102df0 100644 --- a/test_data/compiled-workflows/if_elif_else_with_oneof_parameters.yaml +++ b/test_data/compiled-workflows/if_elif_else_with_oneof_parameters.yaml @@ -70,6 +70,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -78,81 +79,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - outer-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -170,16 +97,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: outer-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -390,70 +345,7 @@ spec: metadata: {} name: comp-condition-4 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - outer-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -474,15 +366,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: outer-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/if_else_with_oneof_artifacts.yaml b/test_data/compiled-workflows/if_else_with_oneof_artifacts.yaml index d7e105ba593..79656b41305 100644 --- a/test_data/compiled-workflows/if_else_with_oneof_artifacts.yaml +++ b/test_data/compiled-workflows/if_else_with_oneof_artifacts.yaml @@ -61,6 +61,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -69,81 +70,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - outer-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -161,16 +88,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: outer-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -347,70 +302,7 @@ spec: metadata: {} name: comp-condition-3 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - outer-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -431,15 +323,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: outer-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/if_else_with_oneof_parameters.yaml b/test_data/compiled-workflows/if_else_with_oneof_parameters.yaml index bac42bafaf2..19d3cc2ab48 100644 --- a/test_data/compiled-workflows/if_else_with_oneof_parameters.yaml +++ b/test_data/compiled-workflows/if_else_with_oneof_parameters.yaml @@ -48,6 +48,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -56,81 +57,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - flip-coin-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -148,16 +75,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: flip-coin-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -336,70 +291,7 @@ spec: metadata: {} name: comp-condition-3 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - flip-coin-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -420,15 +312,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: flip-coin-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/input_artifact.yaml b/test_data/compiled-workflows/input_artifact.yaml index f7f824e1002..940e57ab9c8 100644 --- a/test_data/compiled-workflows/input_artifact.yaml +++ b/test_data/compiled-workflows/input_artifact.yaml @@ -24,6 +24,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -32,81 +33,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - input-artifact - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -124,16 +51,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: input-artifact + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -278,70 +233,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - input-artifact - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -362,15 +254,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: input-artifact + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/iris_pipeline_compiled.yaml b/test_data/compiled-workflows/iris_pipeline_compiled.yaml index 5fbe07a89f1..753424bee68 100644 --- a/test_data/compiled-workflows/iris_pipeline_compiled.yaml +++ b/test_data/compiled-workflows/iris_pipeline_compiled.yaml @@ -77,6 +77,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -85,81 +86,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - iris-training-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -177,16 +104,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: iris-training-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -381,70 +336,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - iris-training-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -465,15 +357,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: iris-training-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/lightweight_python_functions_pipeline.yaml b/test_data/compiled-workflows/lightweight_python_functions_pipeline.yaml index f3a185613bb..6d0c1405de1 100644 --- a/test_data/compiled-workflows/lightweight_python_functions_pipeline.yaml +++ b/test_data/compiled-workflows/lightweight_python_functions_pipeline.yaml @@ -79,6 +79,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -87,81 +88,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - my-test-pipeline-beta - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -179,16 +106,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: my-test-pipeline-beta + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -358,70 +313,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - my-test-pipeline-beta - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -442,15 +334,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: my-test-pipeline-beta + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/lightweight_python_functions_with_outputs.yaml b/test_data/compiled-workflows/lightweight_python_functions_with_outputs.yaml index ec74f4ca372..82ddc820ddb 100644 --- a/test_data/compiled-workflows/lightweight_python_functions_with_outputs.yaml +++ b/test_data/compiled-workflows/lightweight_python_functions_with_outputs.yaml @@ -71,6 +71,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -79,81 +80,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - functions-with-outputs - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -171,16 +98,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: functions-with-outputs + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -399,70 +354,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - functions-with-outputs - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -483,15 +375,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: functions-with-outputs + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/local_execution_pipeline_with_exit_handler.yaml b/test_data/compiled-workflows/local_execution_pipeline_with_exit_handler.yaml index 5fff5e636b2..faf9c7f63bb 100644 --- a/test_data/compiled-workflows/local_execution_pipeline_with_exit_handler.yaml +++ b/test_data/compiled-workflows/local_execution_pipeline_with_exit_handler.yaml @@ -39,6 +39,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -47,81 +48,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-exit-handler - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -139,16 +66,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-exit-handler + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -325,70 +280,7 @@ spec: metadata: {} name: exit-hook-root-record-exit outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-exit-handler - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -409,15 +301,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-exit-handler + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/local_execution_pipeline_with_ignore_upstream_failure.yaml b/test_data/compiled-workflows/local_execution_pipeline_with_ignore_upstream_failure.yaml index a3ce9856048..a22ca988062 100644 --- a/test_data/compiled-workflows/local_execution_pipeline_with_ignore_upstream_failure.yaml +++ b/test_data/compiled-workflows/local_execution_pipeline_with_ignore_upstream_failure.yaml @@ -38,6 +38,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -46,81 +47,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-ignore-upstream-failure - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -138,16 +65,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-ignore-upstream-failure + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -331,70 +286,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-ignore-upstream-failure - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -415,15 +307,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-ignore-upstream-failure + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/local_execution_pipeline_with_k8s_only_methods.yaml b/test_data/compiled-workflows/local_execution_pipeline_with_k8s_only_methods.yaml index a42a1a143d4..23492564be0 100644 --- a/test_data/compiled-workflows/local_execution_pipeline_with_k8s_only_methods.yaml +++ b/test_data/compiled-workflows/local_execution_pipeline_with_k8s_only_methods.yaml @@ -24,6 +24,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -32,81 +33,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-k8s-only-methods - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -124,16 +51,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-k8s-only-methods + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -278,70 +233,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-k8s-only-methods - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -362,15 +254,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-k8s-only-methods + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/local_execution_pipeline_with_oneof.yaml b/test_data/compiled-workflows/local_execution_pipeline_with_oneof.yaml index 9ca78d59eaf..a1d85bbf710 100644 --- a/test_data/compiled-workflows/local_execution_pipeline_with_oneof.yaml +++ b/test_data/compiled-workflows/local_execution_pipeline_with_oneof.yaml @@ -58,6 +58,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -66,81 +67,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-oneof - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -158,16 +85,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-oneof + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -344,70 +299,7 @@ spec: metadata: {} name: comp-condition-3 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-oneof - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -428,15 +320,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-oneof + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/local_execution_pipeline_with_retry.yaml b/test_data/compiled-workflows/local_execution_pipeline_with_retry.yaml index 07405f769a5..239a790d751 100644 --- a/test_data/compiled-workflows/local_execution_pipeline_with_retry.yaml +++ b/test_data/compiled-workflows/local_execution_pipeline_with_retry.yaml @@ -28,6 +28,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -36,81 +37,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-retry - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -128,16 +55,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-retry + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -318,70 +273,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-retry - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -402,15 +294,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-retry + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/log_streaming_compiled.yaml b/test_data/compiled-workflows/log_streaming_compiled.yaml index ebb532ced27..c310b97fd13 100644 --- a/test_data/compiled-workflows/log_streaming_compiled.yaml +++ b/test_data/compiled-workflows/log_streaming_compiled.yaml @@ -28,6 +28,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -36,81 +37,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - log-streaming-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -128,16 +55,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: log-streaming-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -282,70 +237,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - log-streaming-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -366,15 +258,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: log-streaming-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/long-running.yaml b/test_data/compiled-workflows/long-running.yaml index f20fe9eb81e..83712df5940 100644 --- a/test_data/compiled-workflows/long-running.yaml +++ b/test_data/compiled-workflows/long-running.yaml @@ -15,6 +15,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -23,81 +24,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - wait-awhile - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -115,16 +42,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: wait-awhile + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -294,70 +249,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - wait-awhile - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -378,15 +270,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: wait-awhile + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/loop_consume_upstream.yaml b/test_data/compiled-workflows/loop_consume_upstream.yaml index dd511ef15d2..255d759de1f 100644 --- a/test_data/compiled-workflows/loop_consume_upstream.yaml +++ b/test_data/compiled-workflows/loop_consume_upstream.yaml @@ -72,6 +72,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -80,81 +81,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - loop-consume-upstream - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -172,16 +99,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: loop-consume-upstream + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -353,70 +308,7 @@ spec: metadata: {} name: comp-for-loop-1 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - loop-consume-upstream - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -437,15 +329,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: loop-consume-upstream + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/metrics_visualization_v2.yaml b/test_data/compiled-workflows/metrics_visualization_v2.yaml index 908aee03a83..5c3bef58853 100644 --- a/test_data/compiled-workflows/metrics_visualization_v2.yaml +++ b/test_data/compiled-workflows/metrics_visualization_v2.yaml @@ -116,6 +116,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -124,81 +125,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - metrics-visualization-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -216,16 +143,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: metrics-visualization-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -466,70 +421,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - metrics-visualization-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -550,15 +442,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: metrics-visualization-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/missing_kubernetes_optional_inputs.yaml b/test_data/compiled-workflows/missing_kubernetes_optional_inputs.yaml index bdfccd231af..9092e7e9ed3 100644 --- a/test_data/compiled-workflows/missing_kubernetes_optional_inputs.yaml +++ b/test_data/compiled-workflows/missing_kubernetes_optional_inputs.yaml @@ -27,6 +27,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -35,81 +36,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - missing-kubernetes-optional-inputs-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -127,16 +54,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: missing-kubernetes-optional-inputs-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -284,70 +239,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - missing-kubernetes-optional-inputs-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -368,15 +260,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: missing-kubernetes-optional-inputs-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/mixed_parameters.yaml b/test_data/compiled-workflows/mixed_parameters.yaml index 759de56d366..0084eb96106 100644 --- a/test_data/compiled-workflows/mixed_parameters.yaml +++ b/test_data/compiled-workflows/mixed_parameters.yaml @@ -41,6 +41,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -49,81 +50,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - mixed-parameters-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -141,16 +68,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: mixed-parameters-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -295,70 +250,7 @@ spec: metadata: {} name: comp-core outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - mixed-parameters-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -379,15 +271,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: mixed-parameters-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/modelcar.yaml b/test_data/compiled-workflows/modelcar.yaml index 5f85063aec6..12b392372e8 100644 --- a/test_data/compiled-workflows/modelcar.yaml +++ b/test_data/compiled-workflows/modelcar.yaml @@ -49,6 +49,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -57,81 +58,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-modelcar-model - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -149,16 +76,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-modelcar-model + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -409,70 +364,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-modelcar-model - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -493,15 +385,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-modelcar-model + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/mounted_cabundle_configmap.yaml b/test_data/compiled-workflows/mounted_cabundle_configmap.yaml index af20a1ee82a..9d828876635 100644 --- a/test_data/compiled-workflows/mounted_cabundle_configmap.yaml +++ b/test_data/compiled-workflows/mounted_cabundle_configmap.yaml @@ -15,6 +15,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -23,86 +24,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - - --ca_cert_path - - /kfp/certs/ca.crt - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /kfp/certs - name: custom-ca - inputs: + - inputs: parameters: - name: component - name: task @@ -120,27 +42,49 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + ca_cert_path: /kfp/certs/ca.crt + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: echo + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: type: RuntimeDefault - volumes: - - configMap: - items: - - key: test-configmap-key - path: ca.crt - name: test-configmap-name - name: custom-ca - dag: tasks: - arguments: @@ -289,75 +233,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - - --ca_cert_path - - /kfp/certs/ca.crt - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /kfp/certs - name: custom-ca - inputs: + - inputs: parameters: - name: component - default: "" @@ -378,26 +254,47 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + ca_cert_path: /kfp/certs/ca.crt + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: echo + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: type: RuntimeDefault - volumes: - - configMap: - items: - - key: test-configmap-key - path: ca.crt - name: test-configmap-name - name: custom-ca - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/mounted_cabundle_secret.yaml b/test_data/compiled-workflows/mounted_cabundle_secret.yaml index 191b1a9e0e8..75563171663 100644 --- a/test_data/compiled-workflows/mounted_cabundle_secret.yaml +++ b/test_data/compiled-workflows/mounted_cabundle_secret.yaml @@ -15,6 +15,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -23,86 +24,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - - --ca_cert_path - - /kfp/certs/ca.crt - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /kfp/certs - name: custom-ca - inputs: + - inputs: parameters: - name: component - name: task @@ -120,27 +42,49 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + ca_cert_path: /kfp/certs/ca.crt + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: echo + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: type: RuntimeDefault - volumes: - - name: custom-ca - secret: - items: - - key: ca.crt - path: ca.crt - secretName: test-secret-name - dag: tasks: - arguments: @@ -289,75 +233,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - - --ca_cert_path - - /kfp/certs/ca.crt - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /kfp/certs - name: custom-ca - inputs: + - inputs: parameters: - name: component - default: "" @@ -378,26 +254,47 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + ca_cert_path: /kfp/certs/ca.crt + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: echo + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: type: RuntimeDefault - volumes: - - name: custom-ca - secret: - items: - - key: ca.crt - path: ca.crt - secretName: test-secret-name - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/multiple_artifacts_namedtuple.yaml b/test_data/compiled-workflows/multiple_artifacts_namedtuple.yaml index 0f7888c032e..35b5c8d879d 100644 --- a/test_data/compiled-workflows/multiple_artifacts_namedtuple.yaml +++ b/test_data/compiled-workflows/multiple_artifacts_namedtuple.yaml @@ -45,6 +45,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -53,81 +54,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - multiple-artifacts-namedtuple-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -145,16 +72,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: multiple-artifacts-namedtuple-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -299,70 +254,7 @@ spec: metadata: {} name: comp-core outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - multiple-artifacts-namedtuple-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -383,15 +275,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: multiple-artifacts-namedtuple-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/multiple_parameters_namedtuple.yaml b/test_data/compiled-workflows/multiple_parameters_namedtuple.yaml index 80250a01eb4..e21048a8202 100644 --- a/test_data/compiled-workflows/multiple_parameters_namedtuple.yaml +++ b/test_data/compiled-workflows/multiple_parameters_namedtuple.yaml @@ -44,6 +44,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -52,81 +53,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - multiple-parameters-namedtuple-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -144,16 +71,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: multiple-parameters-namedtuple-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -298,70 +253,7 @@ spec: metadata: {} name: comp-core outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - multiple-parameters-namedtuple-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -382,15 +274,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: multiple-parameters-namedtuple-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/nested_parallel_for_secret.yaml b/test_data/compiled-workflows/nested_parallel_for_secret.yaml index cafdaafea00..ae850aeef1e 100644 --- a/test_data/compiled-workflows/nested_parallel_for_secret.yaml +++ b/test_data/compiled-workflows/nested_parallel_for_secret.yaml @@ -45,6 +45,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -53,81 +54,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-4-nested-parallel-for-secret - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -145,16 +72,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-4-nested-parallel-for-secret + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -301,70 +256,7 @@ spec: metadata: {} name: comp-for-loop-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-4-nested-parallel-for-secret - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -385,15 +277,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-4-nested-parallel-for-secret + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/nested_pipeline_opt_input_child_level_compiled.yaml b/test_data/compiled-workflows/nested_pipeline_opt_input_child_level_compiled.yaml index ea49f9f4ad1..8f30369ca93 100644 --- a/test_data/compiled-workflows/nested_pipeline_opt_input_child_level_compiled.yaml +++ b/test_data/compiled-workflows/nested_pipeline_opt_input_child_level_compiled.yaml @@ -104,6 +104,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -112,81 +113,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - nested-pipeline-opt-input-child-level - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -204,16 +131,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: nested-pipeline-opt-input-child-level + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -478,70 +433,7 @@ spec: metadata: {} name: comp-nested-pipeline outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - nested-pipeline-opt-input-child-level - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -562,15 +454,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: nested-pipeline-opt-input-child-level + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml b/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml index 0990d07101f..2a51abdb820 100644 --- a/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml +++ b/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml @@ -55,6 +55,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -63,81 +64,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - nested-pipeline-opt-inputs-nil - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -155,16 +82,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: nested-pipeline-opt-inputs-nil + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -357,70 +312,7 @@ spec: metadata: {} name: comp-nested-pipeline outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - nested-pipeline-opt-inputs-nil - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -441,15 +333,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: nested-pipeline-opt-inputs-nil + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/nested_pipeline_opt_inputs_parent_level_compiled.yaml b/test_data/compiled-workflows/nested_pipeline_opt_inputs_parent_level_compiled.yaml index ba852795df5..8d2db4d7c6e 100644 --- a/test_data/compiled-workflows/nested_pipeline_opt_inputs_parent_level_compiled.yaml +++ b/test_data/compiled-workflows/nested_pipeline_opt_inputs_parent_level_compiled.yaml @@ -107,6 +107,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -115,81 +116,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - nested-pipeline-opt-inputs-parent-level - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -207,16 +134,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: nested-pipeline-opt-inputs-parent-level + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -489,70 +444,7 @@ spec: metadata: {} name: comp-nested-pipeline-non-nil-defaults outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - nested-pipeline-opt-inputs-parent-level - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -573,15 +465,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: nested-pipeline-opt-inputs-parent-level + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/nested_return.yaml b/test_data/compiled-workflows/nested_return.yaml index afefd925131..3557ca841ac 100644 --- a/test_data/compiled-workflows/nested_return.yaml +++ b/test_data/compiled-workflows/nested_return.yaml @@ -25,6 +25,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -33,81 +34,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - nested-return - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -125,16 +52,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: nested-return + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -279,70 +234,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - nested-return - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -363,15 +255,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: nested-return + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/nested_with_parameters.yaml b/test_data/compiled-workflows/nested_with_parameters.yaml index 033f1a45129..ba34ea017dc 100644 --- a/test_data/compiled-workflows/nested_with_parameters.yaml +++ b/test_data/compiled-workflows/nested_with_parameters.yaml @@ -57,6 +57,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -65,81 +66,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -157,16 +84,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: math-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -360,70 +315,7 @@ spec: metadata: {} name: comp-for-loop-4 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -444,15 +336,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: math-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/notebook_component_mixed.yaml b/test_data/compiled-workflows/notebook_component_mixed.yaml index f20b92ce2ec..b7522c0fe55 100644 --- a/test_data/compiled-workflows/notebook_component_mixed.yaml +++ b/test_data/compiled-workflows/notebook_component_mixed.yaml @@ -253,6 +253,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -261,81 +262,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - nb-mixed - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -353,16 +280,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: nb-mixed + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -557,70 +512,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - nb-mixed - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -641,15 +533,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: nb-mixed + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/notebook_component_simple.yaml b/test_data/compiled-workflows/notebook_component_simple.yaml index 8635ad128f7..f6758dbb2a1 100644 --- a/test_data/compiled-workflows/notebook_component_simple.yaml +++ b/test_data/compiled-workflows/notebook_component_simple.yaml @@ -122,6 +122,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -130,81 +131,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - nb-simple - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -222,16 +149,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: nb-simple + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -376,70 +331,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - nb-simple - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -460,15 +352,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: nb-simple + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/output_metrics.yaml b/test_data/compiled-workflows/output_metrics.yaml index a62a2af6f91..61a3431ccac 100644 --- a/test_data/compiled-workflows/output_metrics.yaml +++ b/test_data/compiled-workflows/output_metrics.yaml @@ -26,6 +26,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -34,81 +35,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - output-metrics - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -126,16 +53,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: output-metrics + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -280,70 +235,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - output-metrics - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -364,15 +256,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: output-metrics + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/parallel_for_after_dependency.yaml b/test_data/compiled-workflows/parallel_for_after_dependency.yaml index 5776ce6f02c..c8395a54bed 100644 --- a/test_data/compiled-workflows/parallel_for_after_dependency.yaml +++ b/test_data/compiled-workflows/parallel_for_after_dependency.yaml @@ -27,6 +27,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -35,81 +36,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - loop-with-after-dependency-set - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -127,16 +54,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: loop-with-after-dependency-set + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -281,70 +236,7 @@ spec: metadata: {} name: comp-for-loop-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - loop-with-after-dependency-set - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -365,15 +257,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: loop-with-after-dependency-set + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/parallel_for_mount_pvc.yaml b/test_data/compiled-workflows/parallel_for_mount_pvc.yaml index 926d5f6a646..593299a7ca0 100644 --- a/test_data/compiled-workflows/parallel_for_mount_pvc.yaml +++ b/test_data/compiled-workflows/parallel_for_mount_pvc.yaml @@ -51,6 +51,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -59,81 +60,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-parallel-for-mount-pvc - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -151,16 +78,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-parallel-for-mount-pvc + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -307,70 +262,7 @@ spec: metadata: {} name: comp-for-loop-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-parallel-for-mount-pvc - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -391,15 +283,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-parallel-for-mount-pvc + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/parallel_for_secret.yaml b/test_data/compiled-workflows/parallel_for_secret.yaml index 95c5ce528e6..2382f16c882 100644 --- a/test_data/compiled-workflows/parallel_for_secret.yaml +++ b/test_data/compiled-workflows/parallel_for_secret.yaml @@ -31,6 +31,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -39,81 +40,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-2-parallel-for-secret - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -131,16 +58,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-2-parallel-for-secret + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -287,70 +242,7 @@ spec: metadata: {} name: comp-for-loop-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-2-parallel-for-secret - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -371,15 +263,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-2-parallel-for-secret + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/parameter_cache.yaml b/test_data/compiled-workflows/parameter_cache.yaml index 0d874d1e1ab..62cfcc586bb 100644 --- a/test_data/compiled-workflows/parameter_cache.yaml +++ b/test_data/compiled-workflows/parameter_cache.yaml @@ -41,6 +41,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -49,81 +50,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - parameter-cache-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -141,16 +68,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: parameter-cache-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -295,70 +250,7 @@ spec: metadata: {} name: comp-core outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - parameter-cache-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -379,15 +271,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: parameter-cache-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/parameter_oneof.yaml b/test_data/compiled-workflows/parameter_oneof.yaml index df9c5ac6f90..537cf60414c 100644 --- a/test_data/compiled-workflows/parameter_oneof.yaml +++ b/test_data/compiled-workflows/parameter_oneof.yaml @@ -80,6 +80,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -88,81 +89,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - parameter-oneof-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -180,16 +107,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: parameter-oneof-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -368,70 +323,7 @@ spec: metadata: {} name: comp-condition-3 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - parameter-oneof-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -452,15 +344,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: parameter-oneof-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/parameters_complex.yaml b/test_data/compiled-workflows/parameters_complex.yaml index ba547b6933e..5f9fe0d9da4 100644 --- a/test_data/compiled-workflows/parameters_complex.yaml +++ b/test_data/compiled-workflows/parameters_complex.yaml @@ -74,6 +74,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -82,81 +83,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -174,16 +101,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: math-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -328,70 +283,7 @@ spec: metadata: {} name: comp-for-loop-4 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -412,15 +304,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: math-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/parameters_simple.yaml b/test_data/compiled-workflows/parameters_simple.yaml index 5598d7ca741..ef41b9c8a1d 100644 --- a/test_data/compiled-workflows/parameters_simple.yaml +++ b/test_data/compiled-workflows/parameters_simple.yaml @@ -45,6 +45,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -53,81 +54,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -145,16 +72,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: math-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -299,70 +254,7 @@ spec: metadata: {} name: comp-for-loop-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -383,15 +275,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: math-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_as_exit_task.yaml b/test_data/compiled-workflows/pipeline_as_exit_task.yaml index c54aa3afcf3..d39513682fd 100644 --- a/test_data/compiled-workflows/pipeline_as_exit_task.yaml +++ b/test_data/compiled-workflows/pipeline_as_exit_task.yaml @@ -61,6 +61,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -69,81 +70,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-task-final-status-conditional - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -161,16 +88,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-task-final-status-conditional + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -316,70 +271,7 @@ spec: metadata: {} name: comp-condition-1 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-task-final-status-conditional - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -400,15 +292,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-task-final-status-conditional + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_in_pipeline.yaml b/test_data/compiled-workflows/pipeline_in_pipeline.yaml index 2042a6496cb..c0bca2ac39e 100644 --- a/test_data/compiled-workflows/pipeline_in_pipeline.yaml +++ b/test_data/compiled-workflows/pipeline_in_pipeline.yaml @@ -30,6 +30,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -38,81 +39,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-in-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -130,16 +57,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-in-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -309,70 +264,7 @@ spec: metadata: {} name: comp-inner-pipeline outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-in-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -393,15 +285,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-in-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml b/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml index 81798ae1c4c..a3d4fbacfad 100644 --- a/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml +++ b/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml @@ -39,6 +39,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -47,81 +48,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-in-pipeline-complex - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -139,16 +66,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-in-pipeline-complex + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -325,70 +280,7 @@ spec: metadata: {} name: comp-condition-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-in-pipeline-complex - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -409,15 +301,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-in-pipeline-complex + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_in_pipeline_loaded_from_yaml.yaml b/test_data/compiled-workflows/pipeline_in_pipeline_loaded_from_yaml.yaml index 50a866f8c90..f4814290ff8 100644 --- a/test_data/compiled-workflows/pipeline_in_pipeline_loaded_from_yaml.yaml +++ b/test_data/compiled-workflows/pipeline_in_pipeline_loaded_from_yaml.yaml @@ -46,6 +46,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -54,81 +55,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-in-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -146,16 +73,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-in-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -325,70 +280,7 @@ spec: metadata: {} name: comp-inner-pipeline outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-in-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -409,15 +301,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-in-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_producer_consumer.yaml b/test_data/compiled-workflows/pipeline_producer_consumer.yaml index d8e486cc89b..b1934faebb0 100644 --- a/test_data/compiled-workflows/pipeline_producer_consumer.yaml +++ b/test_data/compiled-workflows/pipeline_producer_consumer.yaml @@ -77,6 +77,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -85,81 +86,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -177,16 +104,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: math-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -331,70 +286,7 @@ spec: metadata: {} name: comp-for-loop-2-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -415,15 +307,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: math-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_after.yaml b/test_data/compiled-workflows/pipeline_with_after.yaml index 3912ec5fcbb..395fac9287e 100644 --- a/test_data/compiled-workflows/pipeline_with_after.yaml +++ b/test_data/compiled-workflows/pipeline_with_after.yaml @@ -18,6 +18,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -26,81 +27,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-after - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -118,16 +45,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-after + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -325,70 +280,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-after - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -409,15 +301,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-after + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_artifact_custom_path.yaml b/test_data/compiled-workflows/pipeline_with_artifact_custom_path.yaml index 98939fb15d0..c62f31c352d 100644 --- a/test_data/compiled-workflows/pipeline_with_artifact_custom_path.yaml +++ b/test_data/compiled-workflows/pipeline_with_artifact_custom_path.yaml @@ -46,6 +46,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -54,81 +55,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-custom-path-artifact - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -146,16 +73,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-custom-path-artifact + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -325,70 +280,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-custom-path-artifact - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -409,15 +301,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-custom-path-artifact + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_artifact_upload_download.yaml b/test_data/compiled-workflows/pipeline_with_artifact_upload_download.yaml index 3682568eb4c..7ad390ae61d 100644 --- a/test_data/compiled-workflows/pipeline_with_artifact_upload_download.yaml +++ b/test_data/compiled-workflows/pipeline_with_artifact_upload_download.yaml @@ -45,6 +45,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -53,81 +54,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-datasets - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -145,16 +72,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-datasets + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -324,70 +279,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-datasets - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -408,15 +300,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-datasets + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_caching.yaml b/test_data/compiled-workflows/pipeline_with_caching.yaml index 1bbd513c77c..29bc0bc3c46 100644 --- a/test_data/compiled-workflows/pipeline_with_caching.yaml +++ b/test_data/compiled-workflows/pipeline_with_caching.yaml @@ -24,6 +24,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -32,81 +33,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-caching - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -124,16 +51,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-caching + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -278,70 +233,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-caching - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -362,15 +254,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-caching + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_collected_artifacts.yaml b/test_data/compiled-workflows/pipeline_with_collected_artifacts.yaml index bc2372abefb..b4150af1d8f 100644 --- a/test_data/compiled-workflows/pipeline_with_collected_artifacts.yaml +++ b/test_data/compiled-workflows/pipeline_with_collected_artifacts.yaml @@ -43,6 +43,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -51,81 +52,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-collected-artifacts - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -143,16 +70,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-collected-artifacts + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -297,70 +252,7 @@ spec: metadata: {} name: comp-for-loop-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-collected-artifacts - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -381,15 +273,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-collected-artifacts + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_concat_placeholder.yaml b/test_data/compiled-workflows/pipeline_with_concat_placeholder.yaml index 17fa857645b..b0d8ee4f7f1 100644 --- a/test_data/compiled-workflows/pipeline_with_concat_placeholder.yaml +++ b/test_data/compiled-workflows/pipeline_with_concat_placeholder.yaml @@ -17,6 +17,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -25,81 +26,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - one-step-pipeline-with-concat-placeholder - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -117,16 +44,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: one-step-pipeline-with-concat-placeholder + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -271,70 +226,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - one-step-pipeline-with-concat-placeholder - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -355,15 +247,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: one-step-pipeline-with-concat-placeholder + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_condition.yaml b/test_data/compiled-workflows/pipeline_with_condition.yaml index b4bf744556d..6fa7d0056cd 100644 --- a/test_data/compiled-workflows/pipeline_with_condition.yaml +++ b/test_data/compiled-workflows/pipeline_with_condition.yaml @@ -43,6 +43,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -51,81 +52,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - single-condition-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -143,16 +70,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: single-condition-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -346,70 +301,7 @@ spec: metadata: {} name: comp-condition-1 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - single-condition-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -430,15 +322,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: single-condition-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml b/test_data/compiled-workflows/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml index f8f8556a811..52a9781b8ad 100644 --- a/test_data/compiled-workflows/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml +++ b/test_data/compiled-workflows/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml @@ -140,6 +140,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -148,81 +149,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-dynamic-condition-output - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -240,16 +167,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-dynamic-condition-output + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -394,70 +349,7 @@ spec: metadata: {} name: comp-condition-1 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-dynamic-condition-output - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -478,15 +370,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-dynamic-condition-output + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_dynamic_importer_metadata.yaml b/test_data/compiled-workflows/pipeline_with_dynamic_importer_metadata.yaml index d561b789414..62776718d82 100644 --- a/test_data/compiled-workflows/pipeline_with_dynamic_importer_metadata.yaml +++ b/test_data/compiled-workflows/pipeline_with_dynamic_importer_metadata.yaml @@ -32,6 +32,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -109,81 +110,7 @@ spec: runAsNonRoot: true seccompProfile: type: RuntimeDefault - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-importer - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -201,16 +128,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-importer + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -379,70 +334,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-importer - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -463,15 +355,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-importer + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_dynamic_task_output_custom_training_job.yaml b/test_data/compiled-workflows/pipeline_with_dynamic_task_output_custom_training_job.yaml index cf68933f943..d2c7407045a 100644 --- a/test_data/compiled-workflows/pipeline_with_dynamic_task_output_custom_training_job.yaml +++ b/test_data/compiled-workflows/pipeline_with_dynamic_task_output_custom_training_job.yaml @@ -102,6 +102,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -110,81 +111,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -202,16 +129,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -429,70 +384,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -513,15 +405,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_env.yaml b/test_data/compiled-workflows/pipeline_with_env.yaml index fde5ffa35d5..5d45ba3f2b1 100644 --- a/test_data/compiled-workflows/pipeline_with_env.yaml +++ b/test_data/compiled-workflows/pipeline_with_env.yaml @@ -30,6 +30,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -38,81 +39,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-env - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -130,16 +57,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-env + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -308,70 +263,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-env - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -392,15 +284,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-env + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_exit_handler.yaml b/test_data/compiled-workflows/pipeline_with_exit_handler.yaml index f7cbdb42a3f..e47ce8fc330 100644 --- a/test_data/compiled-workflows/pipeline_with_exit_handler.yaml +++ b/test_data/compiled-workflows/pipeline_with_exit_handler.yaml @@ -42,6 +42,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -50,81 +51,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-exit-handler - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -142,16 +69,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-exit-handler + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -354,70 +309,7 @@ spec: metadata: {} name: exit-hook-root-print-op outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-exit-handler - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -438,15 +330,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-exit-handler + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_google_artifact_type.yaml b/test_data/compiled-workflows/pipeline_with_google_artifact_type.yaml index aef085eeaf5..60f456dbd3a 100644 --- a/test_data/compiled-workflows/pipeline_with_google_artifact_type.yaml +++ b/test_data/compiled-workflows/pipeline_with_google_artifact_type.yaml @@ -50,6 +50,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -127,81 +128,7 @@ spec: runAsNonRoot: true seccompProfile: type: RuntimeDefault - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-google-types - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -219,16 +146,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-google-types + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -410,70 +365,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-google-types - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -494,15 +386,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-google-types + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_importer.yaml b/test_data/compiled-workflows/pipeline_with_importer.yaml index 2c1634e9548..c28a78487d6 100644 --- a/test_data/compiled-workflows/pipeline_with_importer.yaml +++ b/test_data/compiled-workflows/pipeline_with_importer.yaml @@ -40,6 +40,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -117,81 +118,7 @@ spec: runAsNonRoot: true seccompProfile: type: RuntimeDefault - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-importer - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -209,16 +136,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-importer + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -376,70 +331,7 @@ spec: metadata: {} name: comp-condition-1 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-importer - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -460,15 +352,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-importer + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_importer_and_gcpc_types.yaml b/test_data/compiled-workflows/pipeline_with_importer_and_gcpc_types.yaml index f230897198d..716d1c1ad41 100644 --- a/test_data/compiled-workflows/pipeline_with_importer_and_gcpc_types.yaml +++ b/test_data/compiled-workflows/pipeline_with_importer_and_gcpc_types.yaml @@ -19,6 +19,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -27,81 +28,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-importer-and-gcpc-type - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -119,16 +46,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-importer-and-gcpc-type + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -355,70 +310,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-importer-and-gcpc-type - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -439,15 +331,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-importer-and-gcpc-type + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_importer_workspace.yaml b/test_data/compiled-workflows/pipeline_with_importer_workspace.yaml index 81dd2fa7c49..cf66418f77e 100644 --- a/test_data/compiled-workflows/pipeline_with_importer_workspace.yaml +++ b/test_data/compiled-workflows/pipeline_with_importer_workspace.yaml @@ -107,6 +107,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -191,81 +192,7 @@ spec: - name: kfp-workspace persistentVolumeClaim: claimName: '{{workflow.name}}-kfp-workspace' - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-importer-workspace - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -283,16 +210,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-importer-workspace + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -487,70 +442,7 @@ spec: metadata: {} name: comp-import-stage outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-importer-workspace - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -571,15 +463,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-importer-workspace + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_input_status_state.yaml b/test_data/compiled-workflows/pipeline_with_input_status_state.yaml index 1143ae8d57b..a3f76b42d8c 100644 --- a/test_data/compiled-workflows/pipeline_with_input_status_state.yaml +++ b/test_data/compiled-workflows/pipeline_with_input_status_state.yaml @@ -42,6 +42,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -50,81 +51,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - status-state-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -142,16 +69,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: status-state-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -328,70 +283,7 @@ spec: metadata: {} name: exit-hook-root-echo-state outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - status-state-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -412,15 +304,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: status-state-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_loops.yaml b/test_data/compiled-workflows/pipeline_with_loops.yaml index da73914e125..e9527c4b79f 100644 --- a/test_data/compiled-workflows/pipeline_with_loops.yaml +++ b/test_data/compiled-workflows/pipeline_with_loops.yaml @@ -58,6 +58,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -66,81 +67,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-loops - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -158,16 +85,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-loops + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -472,70 +427,7 @@ spec: metadata: {} name: comp-for-loop-4 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-loops - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -556,15 +448,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-loops + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_loops_and_conditions.yaml b/test_data/compiled-workflows/pipeline_with_loops_and_conditions.yaml index 70bb6366691..11a6fcd8549 100644 --- a/test_data/compiled-workflows/pipeline_with_loops_and_conditions.yaml +++ b/test_data/compiled-workflows/pipeline_with_loops_and_conditions.yaml @@ -104,6 +104,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -112,81 +113,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-loops-and-conditions-multi-layers - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -204,16 +131,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-loops-and-conditions-multi-layers + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -422,70 +377,7 @@ spec: metadata: {} name: comp-for-loop-7 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-loops-and-conditions-multi-layers - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -506,15 +398,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-loops-and-conditions-multi-layers + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_metadata_fields.yaml b/test_data/compiled-workflows/pipeline_with_metadata_fields.yaml index a470d72ca21..7386ddec0c2 100644 --- a/test_data/compiled-workflows/pipeline_with_metadata_fields.yaml +++ b/test_data/compiled-workflows/pipeline_with_metadata_fields.yaml @@ -55,6 +55,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -63,81 +64,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - dataset-concatenator - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -155,16 +82,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: dataset-concatenator + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -334,70 +289,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - dataset-concatenator - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -418,15 +310,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: dataset-concatenator + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_metrics_outputs.yaml b/test_data/compiled-workflows/pipeline_with_metrics_outputs.yaml index 5154c2415be..ce19b71c0eb 100644 --- a/test_data/compiled-workflows/pipeline_with_metrics_outputs.yaml +++ b/test_data/compiled-workflows/pipeline_with_metrics_outputs.yaml @@ -29,6 +29,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -37,81 +38,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-metrics-outputs - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -129,16 +56,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-metrics-outputs + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -283,70 +238,7 @@ spec: metadata: {} name: comp-for-loop-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-metrics-outputs - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -367,15 +259,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-metrics-outputs + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_multiple_exit_handlers.yaml b/test_data/compiled-workflows/pipeline_with_multiple_exit_handlers.yaml index f73f7d0a3f9..ec930914718 100644 --- a/test_data/compiled-workflows/pipeline_with_multiple_exit_handlers.yaml +++ b/test_data/compiled-workflows/pipeline_with_multiple_exit_handlers.yaml @@ -48,6 +48,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -56,81 +57,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-multiple-exit-handlers - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -148,16 +75,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-multiple-exit-handlers + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -490,70 +445,7 @@ spec: metadata: {} name: exit-hook-root-print-op-5 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-multiple-exit-handlers - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -574,15 +466,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-multiple-exit-handlers + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_nested_conditions.yaml b/test_data/compiled-workflows/pipeline_with_nested_conditions.yaml index 21b88b2684f..28de78552a1 100644 --- a/test_data/compiled-workflows/pipeline_with_nested_conditions.yaml +++ b/test_data/compiled-workflows/pipeline_with_nested_conditions.yaml @@ -45,6 +45,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -53,81 +54,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - nested-conditions-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -145,16 +72,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: nested-conditions-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -324,70 +279,7 @@ spec: metadata: {} name: comp-condition-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - nested-conditions-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -408,15 +300,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: nested-conditions-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_nested_conditions_yaml.yaml b/test_data/compiled-workflows/pipeline_with_nested_conditions_yaml.yaml index 2d7853adffb..d646134bf81 100644 --- a/test_data/compiled-workflows/pipeline_with_nested_conditions_yaml.yaml +++ b/test_data/compiled-workflows/pipeline_with_nested_conditions_yaml.yaml @@ -57,6 +57,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -65,81 +66,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - conditional-execution-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -157,16 +84,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: conditional-execution-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -347,70 +302,7 @@ spec: metadata: {} name: comp-condition-3 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - conditional-execution-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -431,15 +323,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: conditional-execution-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_nested_loops.yaml b/test_data/compiled-workflows/pipeline_with_nested_loops.yaml index 57b1bfbf194..2b42f24ac46 100644 --- a/test_data/compiled-workflows/pipeline_with_nested_loops.yaml +++ b/test_data/compiled-workflows/pipeline_with_nested_loops.yaml @@ -35,6 +35,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -43,81 +44,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-nested-loops - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -135,16 +62,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-nested-loops + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -289,70 +244,7 @@ spec: metadata: {} name: comp-for-loop-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-nested-loops - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -373,15 +265,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-nested-loops + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_only_display_name.yaml b/test_data/compiled-workflows/pipeline_with_only_display_name.yaml index fec2b482f4f..7d3530c3d9a 100644 --- a/test_data/compiled-workflows/pipeline_with_only_display_name.yaml +++ b/test_data/compiled-workflows/pipeline_with_only_display_name.yaml @@ -15,6 +15,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -23,81 +24,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - echo-name - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -115,16 +42,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: echo-name + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -269,70 +224,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - echo-name - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -353,15 +245,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: echo-name + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_outputs.yaml b/test_data/compiled-workflows/pipeline_with_outputs.yaml index 099c71c3d2a..1bfa396d235 100644 --- a/test_data/compiled-workflows/pipeline_with_outputs.yaml +++ b/test_data/compiled-workflows/pipeline_with_outputs.yaml @@ -31,6 +31,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -39,81 +40,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-in-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -131,16 +58,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-in-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -310,70 +265,7 @@ spec: metadata: {} name: comp-inner-pipeline outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-in-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -394,15 +286,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-in-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_parallelfor_parallelism.yaml b/test_data/compiled-workflows/pipeline_with_parallelfor_parallelism.yaml index f3750fa4972..797cb991b1f 100644 --- a/test_data/compiled-workflows/pipeline_with_parallelfor_parallelism.yaml +++ b/test_data/compiled-workflows/pipeline_with_parallelfor_parallelism.yaml @@ -122,6 +122,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -130,81 +131,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-loops - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -222,16 +149,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-loops + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -376,70 +331,7 @@ spec: metadata: {} name: comp-for-loop-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-loops - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -460,15 +352,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-loops + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_parallelfor_pipeline_param.yaml b/test_data/compiled-workflows/pipeline_with_parallelfor_pipeline_param.yaml index 590070fafb2..148b8cb139d 100644 --- a/test_data/compiled-workflows/pipeline_with_parallelfor_pipeline_param.yaml +++ b/test_data/compiled-workflows/pipeline_with_parallelfor_pipeline_param.yaml @@ -47,6 +47,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -55,81 +56,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-parallelfor-pipeline-param - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -147,16 +74,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-parallelfor-pipeline-param + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -365,70 +320,7 @@ spec: metadata: {} name: comp-for-loop-5 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-parallelfor-pipeline-param - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -449,15 +341,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-parallelfor-pipeline-param + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_params_containing_format.yaml b/test_data/compiled-workflows/pipeline_with_params_containing_format.yaml index 2c29a52640a..e8be9007e55 100644 --- a/test_data/compiled-workflows/pipeline_with_params_containing_format.yaml +++ b/test_data/compiled-workflows/pipeline_with_params_containing_format.yaml @@ -44,6 +44,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -52,81 +53,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-pipelineparam-containing-format - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -144,16 +71,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-pipelineparam-containing-format + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -299,70 +254,7 @@ spec: metadata: {} name: comp-for-loop-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-pipelineparam-containing-format - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -383,15 +275,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-pipelineparam-containing-format + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_placeholders.yaml b/test_data/compiled-workflows/pipeline_with_placeholders.yaml index 7cec62f4b53..3de64a0c8f2 100644 --- a/test_data/compiled-workflows/pipeline_with_placeholders.yaml +++ b/test_data/compiled-workflows/pipeline_with_placeholders.yaml @@ -31,6 +31,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -39,81 +40,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-placeholders - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -131,16 +58,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-placeholders + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -285,70 +240,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-placeholders - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -369,15 +261,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-placeholders + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_pod_metadata.yaml b/test_data/compiled-workflows/pipeline_with_pod_metadata.yaml index 47bed1a657a..6498145359d 100644 --- a/test_data/compiled-workflows/pipeline_with_pod_metadata.yaml +++ b/test_data/compiled-workflows/pipeline_with_pod_metadata.yaml @@ -146,6 +146,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -154,81 +155,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-pod-metadata - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -246,16 +173,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-pod-metadata + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -1139,70 +1094,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-pod-metadata - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -1223,15 +1115,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-pod-metadata + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_retry.yaml b/test_data/compiled-workflows/pipeline_with_retry.yaml index 11f55866a23..9c6aafc3be1 100644 --- a/test_data/compiled-workflows/pipeline_with_retry.yaml +++ b/test_data/compiled-workflows/pipeline_with_retry.yaml @@ -24,6 +24,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -32,81 +33,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - test-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -124,16 +51,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: test-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -314,70 +269,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - test-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -398,15 +290,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: test-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_reused_component.yaml b/test_data/compiled-workflows/pipeline_with_reused_component.yaml index 0ff9b9cedba..6adb15110c1 100644 --- a/test_data/compiled-workflows/pipeline_with_reused_component.yaml +++ b/test_data/compiled-workflows/pipeline_with_reused_component.yaml @@ -24,6 +24,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -32,81 +33,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-reused-component - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -124,16 +51,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-reused-component + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -328,70 +283,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-reused-component - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -412,15 +304,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-reused-component + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml b/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml index 7e2f8625519..b95fad7941d 100644 --- a/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml +++ b/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml @@ -43,6 +43,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -51,81 +52,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-secret-env - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -143,16 +70,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-secret-env + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -324,70 +279,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-secret-env - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -408,15 +300,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-secret-env + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_secret_as_volume.yaml b/test_data/compiled-workflows/pipeline_with_secret_as_volume.yaml index eb023f22197..83d69827a49 100644 --- a/test_data/compiled-workflows/pipeline_with_secret_as_volume.yaml +++ b/test_data/compiled-workflows/pipeline_with_secret_as_volume.yaml @@ -31,6 +31,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -39,81 +40,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-secret-volume - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -131,16 +58,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-secret-volume + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -287,70 +242,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-secret-volume - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -371,15 +263,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-secret-volume + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_string_machine_fields_pipeline_input.yaml b/test_data/compiled-workflows/pipeline_with_string_machine_fields_pipeline_input.yaml index 5ac3a9a795b..92da7930c38 100644 --- a/test_data/compiled-workflows/pipeline_with_string_machine_fields_pipeline_input.yaml +++ b/test_data/compiled-workflows/pipeline_with_string_machine_fields_pipeline_input.yaml @@ -24,6 +24,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -32,81 +33,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -124,16 +51,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -278,70 +233,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -362,15 +254,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml b/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml index 86cee35bd46..50e4c57f06e 100644 --- a/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml +++ b/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml @@ -76,6 +76,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -84,81 +85,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -176,16 +103,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -428,70 +383,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -512,15 +404,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_submit_request.yaml b/test_data/compiled-workflows/pipeline_with_submit_request.yaml index 64dfa4657af..9bf1a098923 100644 --- a/test_data/compiled-workflows/pipeline_with_submit_request.yaml +++ b/test_data/compiled-workflows/pipeline_with_submit_request.yaml @@ -35,6 +35,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -43,81 +44,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-external-request - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -135,16 +62,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-external-request + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -313,70 +268,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-external-request - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -397,15 +289,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-external-request + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_task_final_status.yaml b/test_data/compiled-workflows/pipeline_with_task_final_status.yaml index 72a58330fd2..f0054f7bd82 100644 --- a/test_data/compiled-workflows/pipeline_with_task_final_status.yaml +++ b/test_data/compiled-workflows/pipeline_with_task_final_status.yaml @@ -58,6 +58,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -66,81 +67,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-task-final-status - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -158,16 +85,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-task-final-status + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -369,70 +324,7 @@ spec: metadata: {} name: exit-hook-root-exit-op outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-task-final-status - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -453,15 +345,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-task-final-status + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_task_final_status_yaml.yaml b/test_data/compiled-workflows/pipeline_with_task_final_status_yaml.yaml index e2ec3a77b22..1ab39418846 100644 --- a/test_data/compiled-workflows/pipeline_with_task_final_status_yaml.yaml +++ b/test_data/compiled-workflows/pipeline_with_task_final_status_yaml.yaml @@ -23,6 +23,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -31,81 +32,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-task-final-status-yaml - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -123,16 +50,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-task-final-status-yaml + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -309,70 +264,7 @@ spec: metadata: {} name: exit-hook-root-exit-op outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-task-final-status-yaml - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -393,15 +285,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-task-final-status-yaml + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_task_using_ignore_upstream_failure.yaml b/test_data/compiled-workflows/pipeline_with_task_using_ignore_upstream_failure.yaml index 1b4856bbf25..0fa2e62e1a8 100644 --- a/test_data/compiled-workflows/pipeline_with_task_using_ignore_upstream_failure.yaml +++ b/test_data/compiled-workflows/pipeline_with_task_using_ignore_upstream_failure.yaml @@ -38,6 +38,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -46,81 +47,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - my-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -138,16 +65,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: my-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -331,70 +286,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - my-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -415,15 +307,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: my-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_utils.yaml b/test_data/compiled-workflows/pipeline_with_utils.yaml index e7fb28ee7f9..bdded464b8b 100644 --- a/test_data/compiled-workflows/pipeline_with_utils.yaml +++ b/test_data/compiled-workflows/pipeline_with_utils.yaml @@ -28,6 +28,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -36,81 +37,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-utils - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -128,16 +55,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-utils + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -282,70 +237,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-utils - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -366,15 +258,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-utils + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_various_io_types.yaml b/test_data/compiled-workflows/pipeline_with_various_io_types.yaml index 799eb7d5f2c..a5e50063349 100644 --- a/test_data/compiled-workflows/pipeline_with_various_io_types.yaml +++ b/test_data/compiled-workflows/pipeline_with_various_io_types.yaml @@ -19,6 +19,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -27,81 +28,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-various-types - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -119,16 +46,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-various-types + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -298,70 +253,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-various-types - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -382,15 +274,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-various-types + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_volume.yaml b/test_data/compiled-workflows/pipeline_with_volume.yaml index 458742d257a..de92f41c8ab 100644 --- a/test_data/compiled-workflows/pipeline_with_volume.yaml +++ b/test_data/compiled-workflows/pipeline_with_volume.yaml @@ -72,6 +72,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -80,81 +81,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-volume - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -172,16 +99,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-volume + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -385,70 +340,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-volume - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -469,15 +361,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-volume + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_volume_long_name.yaml b/test_data/compiled-workflows/pipeline_with_volume_long_name.yaml index 56ad3c0906b..8ea46faca0f 100644 --- a/test_data/compiled-workflows/pipeline_with_volume_long_name.yaml +++ b/test_data/compiled-workflows/pipeline_with_volume_long_name.yaml @@ -46,6 +46,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -54,81 +55,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - here-is-a-pipeline-very-long-name - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -146,16 +73,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: here-is-a-pipeline-very-long-name + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -314,70 +269,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - here-is-a-pipeline-very-long-name - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -398,15 +290,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: here-is-a-pipeline-very-long-name + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_volume_no_cache.yaml b/test_data/compiled-workflows/pipeline_with_volume_no_cache.yaml index 1218333a830..2155b97aba2 100644 --- a/test_data/compiled-workflows/pipeline_with_volume_no_cache.yaml +++ b/test_data/compiled-workflows/pipeline_with_volume_no_cache.yaml @@ -72,6 +72,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -80,81 +81,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-volume-no-cache - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -172,16 +99,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-volume-no-cache + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -385,70 +340,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-volume-no-cache - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -469,15 +361,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-volume-no-cache + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_workspace.yaml b/test_data/compiled-workflows/pipeline_with_workspace.yaml index 34a3dd49915..9c2aa52921e 100644 --- a/test_data/compiled-workflows/pipeline_with_workspace.yaml +++ b/test_data/compiled-workflows/pipeline_with_workspace.yaml @@ -48,6 +48,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -56,81 +57,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-workspace - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -148,16 +75,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-workspace + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -327,70 +282,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-workspace - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -411,15 +303,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pipeline-with-workspace + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/placeholder_with_if_placeholder_none_input_value.yaml b/test_data/compiled-workflows/placeholder_with_if_placeholder_none_input_value.yaml index f2bb06458c2..2f94550e113 100644 --- a/test_data/compiled-workflows/placeholder_with_if_placeholder_none_input_value.yaml +++ b/test_data/compiled-workflows/placeholder_with_if_placeholder_none_input_value.yaml @@ -18,6 +18,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -26,81 +27,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - one-step-pipeline-with-if-placeholder-supply-none - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -118,16 +45,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: one-step-pipeline-with-if-placeholder-supply-none + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -272,70 +227,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - one-step-pipeline-with-if-placeholder-supply-none - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -356,15 +248,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: one-step-pipeline-with-if-placeholder-supply-none + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/preprocess.yaml b/test_data/compiled-workflows/preprocess.yaml index 2165b01e6b0..96bf9165aa9 100644 --- a/test_data/compiled-workflows/preprocess.yaml +++ b/test_data/compiled-workflows/preprocess.yaml @@ -46,6 +46,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -54,81 +55,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - preprocess - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -146,16 +73,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: preprocess + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -300,70 +255,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - preprocess - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -384,15 +276,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: preprocess + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/producer_consumer_param_pipeline.yaml b/test_data/compiled-workflows/producer_consumer_param_pipeline.yaml index ce71e5ff290..1da6bdacc41 100644 --- a/test_data/compiled-workflows/producer_consumer_param_pipeline.yaml +++ b/test_data/compiled-workflows/producer_consumer_param_pipeline.yaml @@ -22,6 +22,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -30,81 +31,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - producer-consumer-param-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -122,16 +49,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: producer-consumer-param-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -301,70 +256,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - producer-consumer-param-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -385,15 +277,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: producer-consumer-param-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pvc_mount.yaml b/test_data/compiled-workflows/pvc_mount.yaml index 50fa4a99b05..08e9b31abdf 100644 --- a/test_data/compiled-workflows/pvc_mount.yaml +++ b/test_data/compiled-workflows/pvc_mount.yaml @@ -42,6 +42,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -50,81 +51,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pvc-mount-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -142,16 +69,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pvc-mount-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -325,70 +280,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pvc-mount-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -409,15 +301,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pvc-mount-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml b/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml index 4019446da75..6240a4ad20a 100644 --- a/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml +++ b/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml @@ -35,6 +35,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -112,81 +113,7 @@ spec: runAsNonRoot: true seccompProfile: type: RuntimeDefault - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - make-language-model-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -204,16 +131,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: make-language-model-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -371,70 +326,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - make-language-model-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -455,15 +347,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: make-language-model-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pythonic_artifacts_test_pipeline.yaml b/test_data/compiled-workflows/pythonic_artifacts_test_pipeline.yaml index 8d1a56969ff..ef596677c82 100644 --- a/test_data/compiled-workflows/pythonic_artifacts_test_pipeline.yaml +++ b/test_data/compiled-workflows/pythonic_artifacts_test_pipeline.yaml @@ -43,6 +43,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -51,81 +52,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pythonic-artifacts-test - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -143,16 +70,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pythonic-artifacts-test + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -322,70 +277,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pythonic-artifacts-test - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -406,15 +298,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: pythonic-artifacts-test + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pythonic_artifacts_with_list_of_artifacts.yaml b/test_data/compiled-workflows/pythonic_artifacts_with_list_of_artifacts.yaml index 7cbebbad713..1573718d65f 100644 --- a/test_data/compiled-workflows/pythonic_artifacts_with_list_of_artifacts.yaml +++ b/test_data/compiled-workflows/pythonic_artifacts_with_list_of_artifacts.yaml @@ -44,6 +44,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -52,81 +53,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - make-and-join-datasets - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -144,16 +71,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: make-and-join-datasets + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -298,70 +253,7 @@ spec: metadata: {} name: comp-for-loop-1 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - make-and-join-datasets - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -382,15 +274,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: make-and-join-datasets + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pythonic_artifacts_with_multiple_returns.yaml b/test_data/compiled-workflows/pythonic_artifacts_with_multiple_returns.yaml index 1952102bdc9..155009f4d57 100644 --- a/test_data/compiled-workflows/pythonic_artifacts_with_multiple_returns.yaml +++ b/test_data/compiled-workflows/pythonic_artifacts_with_multiple_returns.yaml @@ -49,6 +49,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -57,81 +58,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - split-datasets-and-return-first - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -149,16 +76,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: split-datasets-and-return-first + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -303,70 +258,7 @@ spec: metadata: {} name: comp-splitter-pipeline outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - split-datasets-and-return-first - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -387,15 +279,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: split-datasets-and-return-first + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/ray_integration_compiled.yaml b/test_data/compiled-workflows/ray_integration_compiled.yaml index d495579ff62..28fe577f07c 100644 --- a/test_data/compiled-workflows/ray_integration_compiled.yaml +++ b/test_data/compiled-workflows/ray_integration_compiled.yaml @@ -89,6 +89,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -97,81 +98,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - ray-integration-test - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -189,16 +116,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: ray-integration-test + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -343,70 +298,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - ray-integration-test - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -427,15 +319,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: ray-integration-test + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/run_as_user_cache_disabled.yaml b/test_data/compiled-workflows/run_as_user_cache_disabled.yaml index 2ed37911c49..3fec65b9760 100644 --- a/test_data/compiled-workflows/run_as_user_cache_disabled.yaml +++ b/test_data/compiled-workflows/run_as_user_cache_disabled.yaml @@ -15,6 +15,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -24,83 +25,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - - --log_level - - "3" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -118,16 +43,45 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "3" + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: echo + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -274,72 +228,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - - --log_level - - "3" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -360,15 +249,43 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "3" + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: echo + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/run_as_user_cache_enabled.yaml b/test_data/compiled-workflows/run_as_user_cache_enabled.yaml index 2ed37911c49..3fec65b9760 100644 --- a/test_data/compiled-workflows/run_as_user_cache_enabled.yaml +++ b/test_data/compiled-workflows/run_as_user_cache_enabled.yaml @@ -15,6 +15,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -24,83 +25,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - - --log_level - - "3" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -118,16 +43,45 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "3" + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: echo + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -274,72 +228,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - - --log_level - - "3" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -360,15 +249,43 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "3" + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: echo + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/sequential_v1.yaml b/test_data/compiled-workflows/sequential_v1.yaml index aa05a6bd410..14a46f2a5b6 100644 --- a/test_data/compiled-workflows/sequential_v1.yaml +++ b/test_data/compiled-workflows/sequential_v1.yaml @@ -15,6 +15,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -23,81 +24,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - sequential - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -115,16 +42,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: sequential + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -293,70 +248,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - sequential - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -377,15 +269,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: sequential + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/sequential_v2.yaml b/test_data/compiled-workflows/sequential_v2.yaml index d5dd44ca770..f9bc99f4168 100644 --- a/test_data/compiled-workflows/sequential_v2.yaml +++ b/test_data/compiled-workflows/sequential_v2.yaml @@ -19,6 +19,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -27,81 +28,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - sequential - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -119,16 +46,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: sequential + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -298,70 +253,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - sequential - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -382,15 +274,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: sequential + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/take_nap_compiled.yaml b/test_data/compiled-workflows/take_nap_compiled.yaml index 0f94165371c..8b46df45712 100644 --- a/test_data/compiled-workflows/take_nap_compiled.yaml +++ b/test_data/compiled-workflows/take_nap_compiled.yaml @@ -40,6 +40,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -48,81 +49,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - take-nap-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -140,16 +67,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: take-nap-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -319,70 +274,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - take-nap-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -403,15 +295,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: take-nap-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/take_nap_pipeline_root_compiled.yaml b/test_data/compiled-workflows/take_nap_pipeline_root_compiled.yaml index 0f94165371c..8b46df45712 100644 --- a/test_data/compiled-workflows/take_nap_pipeline_root_compiled.yaml +++ b/test_data/compiled-workflows/take_nap_pipeline_root_compiled.yaml @@ -40,6 +40,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -48,81 +49,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - take-nap-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -140,16 +67,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: take-nap-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -319,70 +274,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - take-nap-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -403,15 +295,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: take-nap-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/two_step_pipeline.yaml b/test_data/compiled-workflows/two_step_pipeline.yaml index f6b9c888276..2f20804e8eb 100644 --- a/test_data/compiled-workflows/two_step_pipeline.yaml +++ b/test_data/compiled-workflows/two_step_pipeline.yaml @@ -20,6 +20,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -28,81 +29,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - simple-two-step-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -120,16 +47,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: simple-two-step-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -299,70 +254,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - simple-two-step-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -383,15 +275,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: simple-two-step-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/two_step_pipeline_containerized.yaml b/test_data/compiled-workflows/two_step_pipeline_containerized.yaml index 27f8ea4d398..d0a94a7556e 100644 --- a/test_data/compiled-workflows/two_step_pipeline_containerized.yaml +++ b/test_data/compiled-workflows/two_step_pipeline_containerized.yaml @@ -21,6 +21,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -29,81 +30,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - containerized-two-step-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -121,16 +48,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: containerized-two-step-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -300,70 +255,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - containerized-two-step-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -384,15 +276,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: containerized-two-step-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/upload_download_compiled.yaml b/test_data/compiled-workflows/upload_download_compiled.yaml index 3edca3d8542..abc8f04f4b2 100644 --- a/test_data/compiled-workflows/upload_download_compiled.yaml +++ b/test_data/compiled-workflows/upload_download_compiled.yaml @@ -79,6 +79,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -87,81 +88,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - test-data-passing-pipeline-1 - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -179,16 +106,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: test-data-passing-pipeline-1 + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -383,70 +338,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - test-data-passing-pipeline-1 - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -467,15 +359,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: test-data-passing-pipeline-1 + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/xgboost_sample_pipeline.yaml b/test_data/compiled-workflows/xgboost_sample_pipeline.yaml index 4fcdd6e6a6b..829a0a2516e 100644 --- a/test_data/compiled-workflows/xgboost_sample_pipeline.yaml +++ b/test_data/compiled-workflows/xgboost_sample_pipeline.yaml @@ -283,6 +283,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -291,81 +292,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - xgboost-sample-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -383,16 +310,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: xgboost-sample-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -713,70 +668,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - xgboost-sample-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -797,15 +689,42 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: xgboost-sample-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: