From bb0b8ea9f2a5f0e320bda08d0683656e73ca1e25 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Mon, 22 Sep 2025 21:03:09 +0300 Subject: [PATCH 01/51] Central driver POC https://github.com/kubeflow/pipelines/pull/12023 - Modify Argo compiler: generate a plugin template instead of a container - driver as a http server Signed-off-by: arpechenin --- backend/src/driver/api/request.go | 44 +++ backend/src/driver/api/response.go | 20 ++ .../{v2/cmd => }/driver/execution_paths.go | 0 backend/src/driver/main.go | 178 +++++++++++ backend/src/{v2/cmd => }/driver/main_test.go | 0 backend/src/driver/rpc_handler.go | 285 ++++++++++++++++++ backend/src/v2/compiler/argocompiler/argo.go | 2 +- .../src/v2/compiler/argocompiler/plugin.go | 22 ++ .../base/pipeline/kustomization.yaml | 1 + .../ml-pipeline-driver-plugin-cm.yaml | 23 ++ .../base/pipeline/pipeline-runner-role.yaml | 51 +++- .../base/pipeline/pipeline-runner-sa.yaml | 9 + .../kustomize/env/dev/kustomization.yaml | 4 +- .../application-controller-deployment.yaml | 4 +- 14 files changed, 635 insertions(+), 8 deletions(-) create mode 100644 backend/src/driver/api/request.go create mode 100644 backend/src/driver/api/response.go rename backend/src/{v2/cmd => }/driver/execution_paths.go (100%) create mode 100644 backend/src/driver/main.go rename backend/src/{v2/cmd => }/driver/main_test.go (100%) create mode 100644 backend/src/driver/rpc_handler.go create mode 100644 backend/src/v2/compiler/argocompiler/plugin.go create mode 100644 manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml diff --git a/backend/src/driver/api/request.go b/backend/src/driver/api/request.go new file mode 100644 index 00000000000..08c8360b381 --- /dev/null +++ b/backend/src/driver/api/request.go @@ -0,0 +1,44 @@ +package api + +type DriverPluginArgs struct { + CachedDecisionPath string `json:"cached_decision_path"` + Component string `json:"component,omitempty"` + Container string `json:"container,omitempty"` + RunMetadata string `json:"run_metadata,omitempty"` + DagExecutionID string `json:"dag_execution_id"` + IterationIndex string `json:"iteration_index"` + HttpProxy string `json:"http_proxy"` + HttpsProxy string `json:"https_proxy"` + NoProxy string `json:"no_proxy"` + KubernetesConfig string `json:"kubernetes_config,omitempty"` + RuntimeConfig string `json:"runtime_config,omitempty"` + PipelineName string `json:"pipeline_name"` + RunID string `json:"run_id"` + RunName string `json:"run_name"` + RunDisplayName string `json:"run_display_name"` + TaskName string `json:"task_name"` + Task string `json:"task"` + Type string `json:"type"` + CacheDisabledFlag bool `json:"cache_disabled"` + PublishLogs string `json:"publish_logs"` + ExecutionIdPath string `json:"execution_id_path"` + IterationCountPath string `json:"iteration_count_path"` + ConditionPath string `json:"condition_path"` + PodSpecPathPath string `json:"pod_spec_patch_path"` +} + +type DriverPlugin struct { + DriverPlugin *DriverPluginContainer `json:"driver-plugin"` +} + +type DriverPluginContainer struct { + Args *DriverPluginArgs `json:"args"` +} + +type DriverTemplate struct { + Plugin *DriverPlugin `json:"plugin"` +} + +type DriverRequest struct { + Template *DriverTemplate `json:"template"` +} diff --git a/backend/src/driver/api/response.go b/backend/src/driver/api/response.go new file mode 100644 index 00000000000..fb571609522 --- /dev/null +++ b/backend/src/driver/api/response.go @@ -0,0 +1,20 @@ +package api + +type DriverResponse struct { + Node Node `json:"node"` +} + +type Node struct { + Phase string `json:"phase"` + Outputs Outputs `json:"outputs"` + Message string `json:"message"` +} + +type Outputs struct { + Parameters []Parameter `json:"parameters"` +} + +type Parameter struct { + Name string `json:"name"` + Value string `json:"value"` +} diff --git a/backend/src/v2/cmd/driver/execution_paths.go b/backend/src/driver/execution_paths.go similarity index 100% rename from backend/src/v2/cmd/driver/execution_paths.go rename to backend/src/driver/execution_paths.go diff --git a/backend/src/driver/main.go b/backend/src/driver/main.go new file mode 100644 index 00000000000..985fa224488 --- /dev/null +++ b/backend/src/driver/main.go @@ -0,0 +1,178 @@ +// Copyright 2021-2023 The Kubeflow Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "net/http" + + "google.golang.org/protobuf/encoding/protojson" + + "github.com/kubeflow/pipelines/backend/src/common/util" + + "os" + "path/filepath" + "strconv" + + "github.com/golang/glog" + "github.com/kubeflow/pipelines/backend/src/v2/driver" + "github.com/kubeflow/pipelines/backend/src/v2/metadata" + "github.com/kubeflow/pipelines/kubernetes_platform/go/kubernetesplatform" +) + +const ( + unsetProxyArgValue = "unset" + ROOT_DAG = "ROOT_DAG" + DAG = "DAG" + CONTAINER = "CONTAINER" +) + +var ( + logLevel = flag.String("log_level", "1", "The verbosity level to log.") + + // config + mlmdServerAddress = flag.String("mlmd_server_address", "metadata-grpc-service", "MLMD server address") + mlmdServerPort = flag.String("mlmd_server_port", "8080", "MLMD server port") + + serverPort = flag.String("server_port", ":8080", "Server port") +) + +func main() { + flag.Parse() + + glog.Infof("Setting log level to: '%s'", *logLevel) + err := flag.Set("v", *logLevel) + if err != nil { + glog.Warningf("Failed to set log level: %s", err.Error()) + } + + http.HandleFunc("/api/v1/template.execute", ExecutePlugin) + glog.Infof("Server started at http://localhost%v", *serverPort) + err = http.ListenAndServe(*serverPort, nil) + if err != nil { + glog.Warningf("Failed to start http server: %s", err.Error()) + } +} + +// Use WARNING default logging level to facilitate troubleshooting. +func init() { + flag.Set("logtostderr", "true") + // Change the WARNING to INFO level for debugging. + flag.Set("stderrthreshold", "WARNING") +} + +func parseExecConfigJson(k8sExecConfigJson *string) (*kubernetesplatform.KubernetesExecutorConfig, error) { + var k8sExecCfg *kubernetesplatform.KubernetesExecutorConfig + if *k8sExecConfigJson != "" { + glog.Infof("input kubernetesConfig:%s\n", prettyPrint(*k8sExecConfigJson)) + k8sExecCfg = &kubernetesplatform.KubernetesExecutorConfig{} + if err := util.UnmarshalString(*k8sExecConfigJson, k8sExecCfg); err != nil { + return nil, fmt.Errorf("failed to unmarshal Kubernetes config, error: %w\nKubernetesConfig: %v", err, k8sExecConfigJson) + } + } + return k8sExecCfg, nil +} + +func handleExecution(execution *driver.Execution, driverType string, executionPaths *ExecutionPaths) error { + if execution.ID != 0 { + glog.Infof("output execution.ID=%v", execution.ID) + if executionPaths.ExecutionID != "" { + if err := writeFile(executionPaths.ExecutionID, []byte(fmt.Sprint(execution.ID))); err != nil { + return fmt.Errorf("failed to write execution ID to file: %w", err) + } + } + } + if execution.IterationCount != nil { + if err := writeFile(executionPaths.IterationCount, []byte(fmt.Sprintf("%v", *execution.IterationCount))); err != nil { + return fmt.Errorf("failed to write iteration count to file: %w", err) + } + } else { + if driverType == ROOT_DAG { + if err := writeFile(executionPaths.IterationCount, []byte("0")); err != nil { + return fmt.Errorf("failed to write iteration count to file: %w", err) + } + } + } + if execution.Cached != nil { + if err := writeFile(executionPaths.CachedDecision, []byte(strconv.FormatBool(*execution.Cached))); err != nil { + return fmt.Errorf("failed to write cached decision to file: %w", err) + } + } + if execution.Condition != nil { + if err := writeFile(executionPaths.Condition, []byte(strconv.FormatBool(*execution.Condition))); err != nil { + return fmt.Errorf("failed to write condition to file: %w", err) + } + } else { + // nil is a valid value for Condition + if driverType == ROOT_DAG || driverType == CONTAINER { + if err := writeFile(executionPaths.Condition, []byte("nil")); err != nil { + return fmt.Errorf("failed to write condition to file: %w", err) + } + } + } + if execution.PodSpecPatch != "" { + glog.Infof("output podSpecPatch=\n%s\n", execution.PodSpecPatch) + if executionPaths.PodSpecPatch == "" { + return fmt.Errorf("--pod_spec_patch_path is required for container executor drivers") + } + if err := writeFile(executionPaths.PodSpecPatch, []byte(execution.PodSpecPatch)); err != nil { + return fmt.Errorf("failed to write pod spec patch to file: %w", err) + } + } + if execution.ExecutorInput != nil { + executorInputBytes, err := protojson.Marshal(execution.ExecutorInput) + if err != nil { + return fmt.Errorf("failed to marshal ExecutorInput to JSON: %w", err) + } + executorInputJSON := string(executorInputBytes) + glog.Infof("output ExecutorInput:%s\n", prettyPrint(executorInputJSON)) + } + return nil +} + +func prettyPrint(jsonStr string) string { + var prettyJSON bytes.Buffer + err := json.Indent(&prettyJSON, []byte(jsonStr), "", " ") + if err != nil { + return jsonStr + } + return prettyJSON.String() +} + +func writeFile(path string, data []byte) (err error) { + if path == "" { + return fmt.Errorf("path is not specified") + } + defer func() { + if err != nil { + err = fmt.Errorf("failed to write to %s: %w", path, err) + } + }() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + return os.WriteFile(path, data, 0o644) +} + +func newMlmdClient() (*metadata.Client, error) { + mlmdConfig := metadata.DefaultConfig() + if *mlmdServerAddress != "" && *mlmdServerPort != "" { + mlmdConfig.Address = *mlmdServerAddress + mlmdConfig.Port = *mlmdServerPort + } + return metadata.NewClient(mlmdConfig.Address, mlmdConfig.Port) +} diff --git a/backend/src/v2/cmd/driver/main_test.go b/backend/src/driver/main_test.go similarity index 100% rename from backend/src/v2/cmd/driver/main_test.go rename to backend/src/driver/main_test.go diff --git a/backend/src/driver/rpc_handler.go b/backend/src/driver/rpc_handler.go new file mode 100644 index 00000000000..d80d2180497 --- /dev/null +++ b/backend/src/driver/rpc_handler.go @@ -0,0 +1,285 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "github.com/golang/glog" + "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" + "github.com/kubeflow/pipelines/backend/src/apiserver/config/proxy" + "github.com/kubeflow/pipelines/backend/src/common/util" + "github.com/kubeflow/pipelines/backend/src/driver/api" + "github.com/kubeflow/pipelines/backend/src/v2/cacheutils" + "github.com/kubeflow/pipelines/backend/src/v2/config" + "github.com/kubeflow/pipelines/backend/src/v2/driver" + "google.golang.org/protobuf/encoding/protojson" + "io" + "net/http" + "strconv" +) + +func ExecutePlugin(w http.ResponseWriter, r *http.Request) { + defer func(Body io.ReadCloser) { + err := Body.Close() + if err != nil { + glog.Errorf("Error closing response body: %v", err) + } + }(r.Body) + + if r.Method != http.MethodPost { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + + args, err := parseDriverRequestArgs(r) + if err != nil { + glog.Errorf("Failed to parse driver request args: %v", err) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if args == nil { + glog.Errorf("Failed to parse driver request args: nil") + http.Error(w, "Driver plugin requires at least one argument", http.StatusBadRequest) + return + } + glog.Infof("driver plugin arguments: %v", args) + execution, err := drive(*args) + outputs := extractOutputParameters(execution, args.Type) + if err != nil { + glog.Errorf("unable to drive execution: %v", err) + resp := api.DriverResponse{ + Node: api.Node{ + Phase: "Failed", + Outputs: api.Outputs{ + Parameters: outputs, + }, + Message: fmt.Sprintf("unable to drive execution: %v", err), + }, + } + WriteJSONResponse(w, resp) + return + } + if execution != nil && execution.ExecutorInput != nil { + executorInputBytes, err := protojson.Marshal(execution.ExecutorInput) + if err != nil { + WriteJSONResponse(w, api.DriverResponse{ + Node: api.Node{ + Phase: "Failed", + Outputs: api.Outputs{ + Parameters: outputs, + }, + Message: fmt.Sprintf("unable to drive execution: failed to marshal ExecutorInput to JSON: %v", err), + }, + }) + return + } + executorInputJSON := string(executorInputBytes) + glog.Infof("output ExecutorInput:%s\n", prettyPrint(executorInputJSON)) + } + resp := api.DriverResponse{ + Node: api.Node{ + Phase: "Succeeded", + Outputs: api.Outputs{ + Parameters: outputs, + }, + }, + } + WriteJSONResponse(w, resp) +} + +func parseDriverRequestArgs(r *http.Request) (*api.DriverPluginArgs, error) { + var body api.DriverRequest + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + return nil, fmt.Errorf("failed to parse driver request body: %v", err) + } + if body.Template == nil { + return nil, fmt.Errorf("driver request body.Template is empty") + } else if body.Template.Plugin == nil { + return nil, fmt.Errorf("driver request body.Template.Plugin is empty") + } else if body.Template.Plugin.DriverPlugin == nil { + return nil, fmt.Errorf("driver request body.Template.Plugin.DriverPlugin is empty") + } else if body.Template.Plugin.DriverPlugin.Args == nil { + return nil, fmt.Errorf("driver request body.Template.Plugin.Args is empty") + } + args := body.Template.Plugin.DriverPlugin.Args + if err := validate(*args); err != nil { + return nil, err + } + return body.Template.Plugin.DriverPlugin.Args, nil +} + +func drive(args api.DriverPluginArgs) (execution *driver.Execution, err error) { + defer func() { + if err != nil { + err = fmt.Errorf("KFP driver: %w", err) + } + }() + ctx := context.Background() + + proxy.InitializeConfig(args.HttpProxy, args.HttpsProxy, args.NoProxy) + + glog.Infof("input ComponentSpec:%s\n", prettyPrint(args.Component)) + componentSpec := &pipelinespec.ComponentSpec{} + if err := util.UnmarshalString(args.Component, componentSpec); err != nil { + return nil, fmt.Errorf("failed to unmarshal component spec, error: %w\ncomponentSpec: %v", err, prettyPrint(args.Component)) + } + var taskSpec *pipelinespec.PipelineTaskSpec + if args.Task != "" { + glog.Infof("input TaskSpec:%s\n", prettyPrint(args.Task)) + taskSpec = &pipelinespec.PipelineTaskSpec{} + if err := util.UnmarshalString(args.Task, taskSpec); err != nil { + return nil, fmt.Errorf("failed to unmarshal task spec, error: %w\ntask: %v", err, args.Task) + } + } + + containerSpec := &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{} + if args.Container != "" { + glog.Infof("input ContainerSpec:%s\n", prettyPrint(args.Container)) + if err := util.UnmarshalString(args.Container, containerSpec); err != nil { + return nil, fmt.Errorf("failed to unmarshal container spec, error: %w\ncontainerSpec: %v", err, args.Container) + } + } + var runtimeConfig *pipelinespec.PipelineJob_RuntimeConfig + if args.RuntimeConfig != "" { + glog.Infof("input RuntimeConfig:%s\n", prettyPrint(args.RuntimeConfig)) + runtimeConfig = &pipelinespec.PipelineJob_RuntimeConfig{} + if err := util.UnmarshalString(args.RuntimeConfig, runtimeConfig); err != nil { + return nil, fmt.Errorf("failed to unmarshal runtime config, error: %w\nruntimeConfig: %v", err, args.RuntimeConfig) + } + } + k8sExecCfg, err := parseExecConfigJson(&args.KubernetesConfig) + if err != nil { + return nil, err + } + namespace, err := config.InPodNamespace() + if err != nil { + return nil, err + } + client, err := newMlmdClient() + if err != nil { + return nil, err + } + cacheClient, err := cacheutils.NewClient(args.CacheDisabledFlag) + if err != nil { + return nil, err + } + + dagExecutionID, err := strconv.ParseInt(args.DagExecutionID, 10, 64) + if err != nil { + return nil, fmt.Errorf("failed to parse dag execution id, error: %w", err) + } + iterationIndex, err := strconv.Atoi(args.IterationIndex) + if err != nil { + return nil, fmt.Errorf("failed to parse iteration index, error: %w", err) + } + options := driver.Options{ + PipelineName: args.PipelineName, + RunID: args.RunID, + RunName: args.RunName, + RunDisplayName: args.RunDisplayName, + Namespace: namespace, + Component: componentSpec, + Task: taskSpec, + DAGExecutionID: dagExecutionID, + IterationIndex: iterationIndex, + PublishLogs: args.PublishLogs, + CacheDisabled: args.CacheDisabledFlag, + DriverType: args.Type, + TaskName: args.TaskName, + } + var driverErr error + switch args.Type { + case ROOT_DAG: + options.RuntimeConfig = runtimeConfig + execution, driverErr = driver.RootDAG(ctx, options, client) + case DAG: + execution, driverErr = driver.DAG(ctx, options, client) + case CONTAINER: + options.Container = containerSpec + options.KubernetesExecutorConfig = k8sExecCfg + execution, driverErr = driver.Container(ctx, options, client, cacheClient) + default: + err = fmt.Errorf("unknown driverType %s", args.Type) + } + if driverErr != nil { + if execution == nil { + return nil, driverErr + } + defer func() { + // Override error with driver error, because driver error is more important. + // However, we continue running, because the following code prints debug info that + // may be helpful for figuring out why this failed. + err = driverErr + }() + } + + return execution, nil +} + +func validate(args api.DriverPluginArgs) error { + if args.Type == "" { + return fmt.Errorf("argument type must be specified") + } + if args.HttpProxy == unsetProxyArgValue { + return fmt.Errorf("argument http_proxy is required but can be an empty value") + } + if args.HttpsProxy == unsetProxyArgValue { + return fmt.Errorf("argument https_proxy is required but can be an empty value") + } + if args.NoProxy == unsetProxyArgValue { + return fmt.Errorf("argument no_proxy is required but can be an empty value") + } + // validation responsibility lives in driver itself, so we do not validate all other args + return nil +} + +func extractOutputParameters(execution *driver.Execution, driverType string) []api.Parameter { + if execution == nil { + return []api.Parameter{} + } + var outputs []api.Parameter + if execution.ID != 0 { + outputs = append(outputs, api.Parameter{ + Name: "execution-id", + Value: fmt.Sprint(execution.ID), + }) + } + if execution.IterationCount != nil { + outputs = append(outputs, api.Parameter{ + Name: "iteration-count", + Value: fmt.Sprint(execution.IterationCount), + }) + } else { + if driverType == ROOT_DAG { + outputs = append(outputs, api.Parameter{ + Name: "iteration-count", + Value: fmt.Sprint(0), + }) + } + } + if execution.Cached != nil { + outputs = append(outputs, api.Parameter{ + Name: "cached-decision", + Value: strconv.FormatBool(*execution.Cached), + }) + } + if execution.Condition != nil { + outputs = append(outputs, api.Parameter{ + Name: "condition", + Value: strconv.FormatBool(*execution.Condition), + }) + } + outputs = append(outputs, api.Parameter{ + Name: "pod-spec-patch", + Value: execution.PodSpecPatch, + }) + return outputs +} + +func WriteJSONResponse(w http.ResponseWriter, payload api.DriverResponse) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + if err := json.NewEncoder(w).Encode(payload); err != nil { + http.Error(w, "failed to encode response", http.StatusInternalServerError) + } +} diff --git a/backend/src/v2/compiler/argocompiler/argo.go b/backend/src/v2/compiler/argocompiler/argo.go index 293aee56d65..1977d81d41f 100644 --- a/backend/src/v2/compiler/argocompiler/argo.go +++ b/backend/src/v2/compiler/argocompiler/argo.go @@ -465,7 +465,7 @@ var driverResources = k8score.ResourceRequirements{ // Launcher only copies the binary into the volume, so it needs minimal resources. var launcherResources = k8score.ResourceRequirements{ Limits: map[k8score.ResourceName]k8sres.Quantity{ - k8score.ResourceMemory: k8sres.MustParse("128Mi"), + k8score.ResourceMemory: k8sres.MustParse("256Mi"), k8score.ResourceCPU: k8sres.MustParse("0.5"), }, Requests: map[k8score.ResourceName]k8sres.Quantity{ diff --git a/backend/src/v2/compiler/argocompiler/plugin.go b/backend/src/v2/compiler/argocompiler/plugin.go new file mode 100644 index 00000000000..941f9fcd04b --- /dev/null +++ b/backend/src/v2/compiler/argocompiler/plugin.go @@ -0,0 +1,22 @@ +package argocompiler + +import ( + "encoding/json" + "fmt" + wfapi "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" +) + +func driverPlugin(params map[string]interface{}) (*wfapi.Plugin, error) { + pluginConfig := map[string]interface{}{ + "driver-plugin": map[string]interface{}{ + "args": params, + }, + } + jsonConfig, err := json.Marshal(pluginConfig) + if err != nil { + return nil, fmt.Errorf("driver plugin creation error: marshaling plugin config to JSON failed: %w", err) + } + return &wfapi.Plugin{Object: wfapi.Object{ + Value: jsonConfig, + }}, nil +} diff --git a/manifests/kustomize/base/pipeline/kustomization.yaml b/manifests/kustomize/base/pipeline/kustomization.yaml index 3cec8622afb..7e40aa270e9 100644 --- a/manifests/kustomize/base/pipeline/kustomization.yaml +++ b/manifests/kustomize/base/pipeline/kustomization.yaml @@ -34,6 +34,7 @@ resources: - container-builder-sa.yaml - viewer-sa.yaml - kfp-launcher-configmap.yaml + - ml-pipeline-driver-plugin-cm.yaml - public_configmap_role.yaml - public_configmap_role_binding.yaml configMapGenerator: diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml new file mode 100644 index 00000000000..da703b2c66d --- /dev/null +++ b/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + workflows.argoproj.io/configmap-type: ExecutorPlugin + name: ml-pipeline-driver-agent +data: + sidecar.automountServiceAccountToken: "true" + sidecar.container: | + image: ntny/kfp-driver:central-driver-poc + name: driver-plugin + ports: + - containerPort: 8080 + resources: + limits: + cpu: "1" + memory: 1Gi + requests: + cpu: 250m + memory: 512Mi + securityContext: + runAsNonRoot: true + runAsUser: 65534 diff --git a/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml b/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml index eba0ee9f2d6..8eb2f07beef 100644 --- a/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml +++ b/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml @@ -33,15 +33,19 @@ rules: - delete - get - apiGroups: - - argoproj.io + - argoproj.io resources: - workflows + - workflowtaskresults + - workflowtasksets + - "workflowtasksets/status" verbs: - get - - list + - create + - patch - watch + - list - update - - patch - apiGroups: - "" resources: @@ -85,3 +89,44 @@ rules: verbs: - create - patch + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ml-pipeline-driver-agent-executor-plugin + labels: + application-crd-id: kubeflow-pipelines +secrets: + - name: ml-pipeline-driver-agent-executor-plugin.service-account-token + +--- +apiVersion: v1 +kind: Secret +metadata: + name: ml-pipeline-driver-agent-executor-plugin.service-account-token + annotations: + kubernetes.io/service-account.name: ml-pipeline-driver-agent-executor-plugin +type: kubernetes.io/service-account-token + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: configmap-reader +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch"] # права на чтение +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: configmap-reader-binding +subjects: + - kind: ServiceAccount + name: ml-pipeline-driver-agent-executor-plugin +roleRef: + kind: Role + name: configmap-reader + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/manifests/kustomize/base/pipeline/pipeline-runner-sa.yaml b/manifests/kustomize/base/pipeline/pipeline-runner-sa.yaml index 8cb2c669fb2..e1f42088371 100644 --- a/manifests/kustomize/base/pipeline/pipeline-runner-sa.yaml +++ b/manifests/kustomize/base/pipeline/pipeline-runner-sa.yaml @@ -2,3 +2,12 @@ apiVersion: v1 kind: ServiceAccount metadata: name: pipeline-runner + +--- +apiVersion: v1 +kind: Secret +metadata: + name: pipeline-runner.service-account-token + annotations: + kubernetes.io/service-account.name: pipeline-runner +type: kubernetes.io/service-account-token diff --git a/manifests/kustomize/env/dev/kustomization.yaml b/manifests/kustomize/env/dev/kustomization.yaml index f2ed5ddabf1..72b7adbbf33 100644 --- a/manifests/kustomize/env/dev/kustomization.yaml +++ b/manifests/kustomize/env/dev/kustomization.yaml @@ -9,8 +9,8 @@ resources: - ../gcp/inverse-proxy images: -- name: ghcr.io/kubeflow/kfp-api-server - newTag: master +- name: ntny/kfp-driver + newTag: central-driver-poc - name: ghcr.io/kubeflow/kfp-frontend newTag: master - name: ghcr.io/kubeflow/kfp-persistence-agent diff --git a/manifests/kustomize/third-party/application/application-controller-deployment.yaml b/manifests/kustomize/third-party/application/application-controller-deployment.yaml index 1f1c589aaef..2f4277da116 100644 --- a/manifests/kustomize/third-party/application/application-controller-deployment.yaml +++ b/manifests/kustomize/third-party/application/application-controller-deployment.yaml @@ -31,8 +31,8 @@ spec: resources: limits: cpu: 100m - memory: 30Mi + memory: 128Mi requests: cpu: 100m - memory: 20Mi + memory: 64Mi serviceAccountName: application From 934655c47f169d9e17c40dd3edecd6922f6a85a4 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Thu, 25 Sep 2025 02:22:55 +0300 Subject: [PATCH 02/51] - add pipeline flow details to contribution Signed-off-by: arpechenin --- manifests/kustomize/base/pipeline/pipeline-runner-role.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml b/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml index 8eb2f07beef..833885d0d0e 100644 --- a/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml +++ b/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml @@ -33,7 +33,7 @@ rules: - delete - get - apiGroups: - - argoproj.io + - argoproj.io resources: - workflows - workflowtaskresults From 7e77e29098963ba24955604b695f5cce85568ac1 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Tue, 30 Sep 2025 18:33:14 +0300 Subject: [PATCH 03/51] - rebase brunch - add feature to regenerate all specs Signed-off-by: arpechenin --- backend/test/compiler/matchers/workflow_matcher.go | 2 +- .../pipeline_with_artifact_custom_path.yaml | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/backend/test/compiler/matchers/workflow_matcher.go b/backend/test/compiler/matchers/workflow_matcher.go index 632ff9b4f7c..959a6462c5a 100644 --- a/backend/test/compiler/matchers/workflow_matcher.go +++ b/backend/test/compiler/matchers/workflow_matcher.go @@ -128,7 +128,7 @@ func CompareWorkflows(actual *v1alpha1.Workflow, expected *v1alpha1.Workflow) { gomega.Expect(actual.Spec.Templates[index].HTTP).To(gomega.Equal(template.HTTP), "HTTP is not same") gomega.Expect(actual.Spec.Templates[index].Memoize).To(gomega.Equal(template.Memoize), "Memoize is not same") gomega.Expect(actual.Spec.Templates[index].Metadata).To(gomega.Equal(template.Metadata), "Metadata is not same") - gomega.Expect(actual.Spec.Templates[index].Plugin).To(gomega.Equal(template.Plugin), "Plugin is not same") + // gomega.Expect(actual.Spec.Templates[index].Plugin).To(gomega.Equal(template.Plugin), "Plugin is not same") gomega.Expect(actual.Spec.Templates[index].PriorityClassName).To(gomega.Equal(template.PriorityClassName), "PriorityClassName is not same") gomega.Expect(actual.Spec.Templates[index].Resource).To(gomega.Equal(template.Resource), "Resource is not same") gomega.Expect(actual.Spec.Templates[index].Script).To(gomega.Equal(template.Script), "Script is not same") diff --git a/test_data/compiled-workflows/pipeline_with_artifact_custom_path.yaml b/test_data/compiled-workflows/pipeline_with_artifact_custom_path.yaml index 06a501085f9..4fabd3c4e71 100644 --- a/test_data/compiled-workflows/pipeline_with_artifact_custom_path.yaml +++ b/test_data/compiled-workflows/pipeline_with_artifact_custom_path.yaml @@ -135,16 +135,16 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition - dag: tasks: - arguments: @@ -368,15 +368,15 @@ spec: parameters: - name: execution-id valueFrom: - path: /tmp/outputs/execution-id + jsonPath: $.execution-id - name: iteration-count valueFrom: default: "0" - path: /tmp/outputs/iteration-count + jsonPath: $.iteration-count - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition - dag: tasks: - arguments: From 395c8c17e57a02b409749551368fdbeb1e3ed936 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Tue, 30 Sep 2025 21:24:14 +0300 Subject: [PATCH 04/51] - remove redundant regenerate code Signed-off-by: arpechenin --- backend/test/compiler/matchers/workflow_matcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/test/compiler/matchers/workflow_matcher.go b/backend/test/compiler/matchers/workflow_matcher.go index 959a6462c5a..632ff9b4f7c 100644 --- a/backend/test/compiler/matchers/workflow_matcher.go +++ b/backend/test/compiler/matchers/workflow_matcher.go @@ -128,7 +128,7 @@ func CompareWorkflows(actual *v1alpha1.Workflow, expected *v1alpha1.Workflow) { gomega.Expect(actual.Spec.Templates[index].HTTP).To(gomega.Equal(template.HTTP), "HTTP is not same") gomega.Expect(actual.Spec.Templates[index].Memoize).To(gomega.Equal(template.Memoize), "Memoize is not same") gomega.Expect(actual.Spec.Templates[index].Metadata).To(gomega.Equal(template.Metadata), "Metadata is not same") - // gomega.Expect(actual.Spec.Templates[index].Plugin).To(gomega.Equal(template.Plugin), "Plugin is not same") + gomega.Expect(actual.Spec.Templates[index].Plugin).To(gomega.Equal(template.Plugin), "Plugin is not same") gomega.Expect(actual.Spec.Templates[index].PriorityClassName).To(gomega.Equal(template.PriorityClassName), "PriorityClassName is not same") gomega.Expect(actual.Spec.Templates[index].Resource).To(gomega.Equal(template.Resource), "Resource is not same") gomega.Expect(actual.Spec.Templates[index].Script).To(gomega.Equal(template.Script), "Script is not same") From c2c0a49f0630f0593b1dfacf3ab06a59077b499a Mon Sep 17 00:00:00 2001 From: arpechenin Date: Fri, 14 Nov 2025 00:08:00 +0300 Subject: [PATCH 05/51] - rebase master - fix tests Signed-off-by: arpechenin --- backend/Dockerfile.driver | 2 +- backend/src/driver/api/request.go | 50 ++++++++++--------- backend/src/driver/main.go | 5 +- backend/src/driver/rpc_handler.go | 17 +++++-- backend/src/v2/compiler/argocompiler/argo.go | 2 - backend/test/compiler/utils/workflow_utils.go | 29 ++++++++++- .../ml-pipeline-apiserver-deployment.yaml | 2 +- .../ml-pipeline-driver-plugin-cm.yaml | 2 +- .../workflow-controller-deployment-patch.yaml | 3 ++ .../installs/namespace/kustomization.yaml | 6 +++ .../workflow-controller-argo-role-patch.json | 11 ++++ 11 files changed, 92 insertions(+), 37 deletions(-) create mode 100644 manifests/kustomize/third-party/argo/installs/namespace/workflow-controller-argo-role-patch.json diff --git a/backend/Dockerfile.driver b/backend/Dockerfile.driver index fd62ca91c34..e3a1b4bb01a 100644 --- a/backend/Dockerfile.driver +++ b/backend/Dockerfile.driver @@ -27,7 +27,7 @@ RUN GO111MODULE=on go mod download COPY . . -RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux go build -tags netgo -gcflags="${GCFLAGS}" -ldflags '-extldflags "-static"' -o /bin/driver ./backend/src/v2/cmd/driver/*.go +RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux go build -tags netgo -gcflags="${GCFLAGS}" -ldflags '-extldflags "-static"' -o /bin/driver ./backend/src/driver/*.go FROM alpine:3.21 diff --git a/backend/src/driver/api/request.go b/backend/src/driver/api/request.go index 08c8360b381..1a8cf38fe4d 100644 --- a/backend/src/driver/api/request.go +++ b/backend/src/driver/api/request.go @@ -1,30 +1,32 @@ package api type DriverPluginArgs struct { - CachedDecisionPath string `json:"cached_decision_path"` - Component string `json:"component,omitempty"` - Container string `json:"container,omitempty"` - RunMetadata string `json:"run_metadata,omitempty"` - DagExecutionID string `json:"dag_execution_id"` - IterationIndex string `json:"iteration_index"` - HttpProxy string `json:"http_proxy"` - HttpsProxy string `json:"https_proxy"` - NoProxy string `json:"no_proxy"` - KubernetesConfig string `json:"kubernetes_config,omitempty"` - RuntimeConfig string `json:"runtime_config,omitempty"` - PipelineName string `json:"pipeline_name"` - RunID string `json:"run_id"` - RunName string `json:"run_name"` - RunDisplayName string `json:"run_display_name"` - TaskName string `json:"task_name"` - Task string `json:"task"` - Type string `json:"type"` - CacheDisabledFlag bool `json:"cache_disabled"` - PublishLogs string `json:"publish_logs"` - ExecutionIdPath string `json:"execution_id_path"` - IterationCountPath string `json:"iteration_count_path"` - ConditionPath string `json:"condition_path"` - PodSpecPathPath string `json:"pod_spec_patch_path"` + CachedDecisionPath string `json:"cached_decision_path"` + Component string `json:"component,omitempty"` + Container string `json:"container,omitempty"` + DagExecutionID string `json:"dag_execution_id"` + IterationIndex string `json:"iteration_index"` + HttpProxy string `json:"http_proxy"` + HttpsProxy string `json:"https_proxy"` + NoProxy string `json:"no_proxy"` + KubernetesConfig string `json:"kubernetes_config,omitempty"` + RuntimeConfig string `json:"runtime_config,omitempty"` + PipelineName string `json:"pipeline_name"` + PublishLogs string `json:"publish_logs,omitempty"` + RunID string `json:"run_id"` + RunName string `json:"run_name"` + RunDisplayName string `json:"run_display_name"` + TaskName string `json:"task_name"` + Task string `json:"task"` + Type string `json:"type"` + CacheDisabledFlag bool `json:"cache_disabled"` + ExecutionIdPath string `json:"execution_id_path"` + IterationCountPath string `json:"iteration_count_path"` + ConditionPath string `json:"condition_path"` + PodSpecPathPath string `json:"pod_spec_patch_path"` + MlPipelineTLSEnabled bool `json:"ml_pipeline_tls_enabled"` + MetadataTLSEnabled bool `json:"metadata_tls_enabled"` + CACertPath string `json:"ca_cert_path"` } type DriverPlugin struct { diff --git a/backend/src/driver/main.go b/backend/src/driver/main.go index 985fa224488..1ff38f3bf31 100644 --- a/backend/src/driver/main.go +++ b/backend/src/driver/main.go @@ -15,6 +15,7 @@ package main import ( "bytes" + "crypto/tls" "encoding/json" "flag" "fmt" @@ -168,11 +169,11 @@ func writeFile(path string, data []byte) (err error) { return os.WriteFile(path, data, 0o644) } -func newMlmdClient() (*metadata.Client, error) { +func newMlmdClient(tlsCfg *tls.Config) (*metadata.Client, error) { mlmdConfig := metadata.DefaultConfig() if *mlmdServerAddress != "" && *mlmdServerPort != "" { mlmdConfig.Address = *mlmdServerAddress mlmdConfig.Port = *mlmdServerPort } - return metadata.NewClient(mlmdConfig.Address, mlmdConfig.Port) + return metadata.NewClient(mlmdConfig.Address, mlmdConfig.Port, tlsCfg) } diff --git a/backend/src/driver/rpc_handler.go b/backend/src/driver/rpc_handler.go index d80d2180497..db119ae8d56 100644 --- a/backend/src/driver/rpc_handler.go +++ b/backend/src/driver/rpc_handler.go @@ -2,8 +2,13 @@ package main import ( "context" + "crypto/tls" "encoding/json" "fmt" + "io" + "net/http" + "strconv" + "github.com/golang/glog" "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" "github.com/kubeflow/pipelines/backend/src/apiserver/config/proxy" @@ -13,9 +18,6 @@ import ( "github.com/kubeflow/pipelines/backend/src/v2/config" "github.com/kubeflow/pipelines/backend/src/v2/driver" "google.golang.org/protobuf/encoding/protojson" - "io" - "net/http" - "strconv" ) func ExecutePlugin(w http.ResponseWriter, r *http.Request) { @@ -155,11 +157,16 @@ func drive(args api.DriverPluginArgs) (execution *driver.Execution, err error) { if err != nil { return nil, err } - client, err := newMlmdClient() + var tlsCfg *tls.Config + if args.MetadataTLSEnabled { + tlsCfg, err = util.GetTLSConfig(args.CACertPath) + return nil, fmt.Errorf("unable to drive driver: failed to load TLS configuration: %v", err) + } + client, err := newMlmdClient(tlsCfg) if err != nil { return nil, err } - cacheClient, err := cacheutils.NewClient(args.CacheDisabledFlag) + cacheClient, err := cacheutils.NewClient(args.CacheDisabledFlag, tlsCfg) if err != nil { return nil, err } diff --git a/backend/src/v2/compiler/argocompiler/argo.go b/backend/src/v2/compiler/argocompiler/argo.go index 1977d81d41f..321f301e9db 100644 --- a/backend/src/v2/compiler/argocompiler/argo.go +++ b/backend/src/v2/compiler/argocompiler/argo.go @@ -172,8 +172,6 @@ func Compile(jobArg *pipelinespec.PipelineJob, kubernetesSpecArg *pipelinespec.S // TODO(chensun): release process and update the images. launcherImage: GetLauncherImage(), launcherCommand: GetLauncherCommand(), - driverImage: GetDriverImage(), - driverCommand: GetDriverCommand(), job: job, spec: spec, executors: deploy.GetExecutors(), diff --git a/backend/test/compiler/utils/workflow_utils.go b/backend/test/compiler/utils/workflow_utils.go index 1cb7a8ac5f6..a135cbc69cf 100644 --- a/backend/test/compiler/utils/workflow_utils.go +++ b/backend/test/compiler/utils/workflow_utils.go @@ -36,6 +36,8 @@ import ( v1 "k8s.io/api/core/v1" ) +type driverPlugin map[string]map[string]map[string]interface{} + // LoadPipelineSpecsFromIR - Unmarshall Pipeline Spec IR into a tuple of (pipelinespec.PipelineJob, pipelinespec.SinglePlatformSpec) func LoadPipelineSpecsFromIR(pipelineIRFilePath string, cacheDisabled bool, defaultWorkspace *v1.PersistentVolumeClaimSpec) (*pipelinespec.PipelineJob, *pipelinespec.SinglePlatformSpec) { pipelineSpecsFromFile := testutil.ParseFileToSpecs(pipelineIRFilePath, cacheDisabled, defaultWorkspace) @@ -89,6 +91,31 @@ func CreateCompiledWorkflowFile(compiledWorflow *v1alpha1.Workflow, compiledWork return testutil.CreateFile(compiledWorkflowFilePath, [][]byte{fileContents}) } +func ConfigurePluginSettings(workflow *v1alpha1.Workflow, remove bool) *v1alpha1.Workflow { + configuredWorkflow := workflow.DeepCopy() + for i, template := range configuredWorkflow.Spec.Templates { + if template.Plugin != nil { + var pluginMap driverPlugin + if err := json.Unmarshal(template.Plugin.Value, &pluginMap); err == nil { + if driverPlugin, ok := pluginMap["driver-plugin"]; ok { + if args, ok := driverPlugin["args"]; ok { + if remove { + args["cache_disabled"] = false + } else { + args["cache_disabled"] = true + } + } + } + jsonPlugin, err := json.Marshal(pluginMap) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to marshal plugin map") + configuredWorkflow.Spec.Templates[i].Plugin.Value = jsonPlugin + } + } + } + + return configuredWorkflow +} + // ConfigureCacheSettings - Add/Remove cache_disabled args in the workflow func ConfigureCacheSettings(workflow *v1alpha1.Workflow, remove bool) *v1alpha1.Workflow { cacheDisabledArg := "--cache_disabled" @@ -138,5 +165,5 @@ func ConfigureCacheSettings(workflow *v1alpha1.Workflow, remove bool) *v1alpha1. } } } - return configuredWorkflow + return ConfigurePluginSettings(configuredWorkflow, remove) } diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml index 005dbe42885..6467e3cb988 100644 --- a/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml +++ b/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml @@ -132,7 +132,7 @@ spec: # JSON patch to apply to compiled workflow specifications - name: COMPILED_PIPELINE_SPEC_PATCH value: "{}" - image: ghcr.io/kubeflow/kfp-api-server:dummy + image: ntny/kfp-api-server:central-driver-beta imagePullPolicy: IfNotPresent name: ml-pipeline-api-server ports: diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml index da703b2c66d..66cfc5caabd 100644 --- a/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml +++ b/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml @@ -7,7 +7,7 @@ metadata: data: sidecar.automountServiceAccountToken: "true" sidecar.container: | - image: ntny/kfp-driver:central-driver-poc + image: ntny/kfp-driver:central-driver-beta name: driver-plugin ports: - containerPort: 8080 diff --git a/manifests/kustomize/third-party/argo/base/workflow-controller-deployment-patch.yaml b/manifests/kustomize/third-party/argo/base/workflow-controller-deployment-patch.yaml index abeec455e3d..d41d52359c6 100644 --- a/manifests/kustomize/third-party/argo/base/workflow-controller-deployment-patch.yaml +++ b/manifests/kustomize/third-party/argo/base/workflow-controller-deployment-patch.yaml @@ -17,6 +17,9 @@ spec: - workflow-controller-configmap - --executor-image - quay.io/argoproj/argoexec:v3.7.3 + env: + - name: ARGO_EXECUTOR_PLUGINS + value: "true" securityContext: readOnlyRootFilesystem: true runAsNonRoot: true diff --git a/manifests/kustomize/third-party/argo/installs/namespace/kustomization.yaml b/manifests/kustomize/third-party/argo/installs/namespace/kustomization.yaml index 90b8eea8588..52df92bed5a 100644 --- a/manifests/kustomize/third-party/argo/installs/namespace/kustomization.yaml +++ b/manifests/kustomize/third-party/argo/installs/namespace/kustomization.yaml @@ -16,3 +16,9 @@ patches: kind: Deployment name: workflow-controller version: v1 +- path: workflow-controller-argo-role-patch.json + target: + group: rbac.authorization.k8s.io + version: v1 + kind: Role + name: argo-role \ No newline at end of file diff --git a/manifests/kustomize/third-party/argo/installs/namespace/workflow-controller-argo-role-patch.json b/manifests/kustomize/third-party/argo/installs/namespace/workflow-controller-argo-role-patch.json new file mode 100644 index 00000000000..f0ca733247f --- /dev/null +++ b/manifests/kustomize/third-party/argo/installs/namespace/workflow-controller-argo-role-patch.json @@ -0,0 +1,11 @@ +[ + { + "op": "add", + "path": "/rules/-", + "value": { + "apiGroups": ["argoproj.io"], + "resources": ["workflowtasksets/status"], + "verbs": ["get","list","watch","update","patch","delete","create"] + } + } +] \ No newline at end of file From 037436f61c50a63b2c65ed01e9215d62f307d8ab Mon Sep 17 00:00:00 2001 From: arpechenin Date: Mon, 17 Nov 2025 23:22:56 +0300 Subject: [PATCH 06/51] - merge tests - fix formatting Signed-off-by: arpechenin --- backend/src/driver/api/request.go | 21 +- backend/src/driver/api/response.go | 14 + backend/src/driver/execution_paths.go | 14 + backend/src/driver/main.go | 17 +- backend/src/driver/main_test.go | 18 +- backend/src/driver/rpc_handler.go | 94 +++-- .../src/v2/compiler/argocompiler/container.go | 119 +++---- backend/src/v2/compiler/argocompiler/dag.go | 97 +++-- .../src/v2/compiler/argocompiler/plugin.go | 15 + .../ml-pipeline-apiserver-deployment.yaml | 2 +- .../ml-pipeline-driver-plugin-cm.yaml | 2 +- .../kustomize/env/dev/kustomization.yaml | 4 +- .../arguments-parameters.yaml | 334 ------------------ .../arguments.pipeline.yaml | 334 ------------------ test_data/compiled-workflows/hello-world.yaml | 334 ------------------ 15 files changed, 244 insertions(+), 1175 deletions(-) delete mode 100644 test_data/compiled-workflows/arguments-parameters.yaml delete mode 100644 test_data/compiled-workflows/arguments.pipeline.yaml delete mode 100644 test_data/compiled-workflows/hello-world.yaml diff --git a/backend/src/driver/api/request.go b/backend/src/driver/api/request.go index 1a8cf38fe4d..0da025e4bb4 100644 --- a/backend/src/driver/api/request.go +++ b/backend/src/driver/api/request.go @@ -1,3 +1,18 @@ +// Copyright 2025 The Kubeflow Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package api provides HTTP DTOs used by the driver server. package api type DriverPluginArgs struct { @@ -6,8 +21,8 @@ type DriverPluginArgs struct { Container string `json:"container,omitempty"` DagExecutionID string `json:"dag_execution_id"` IterationIndex string `json:"iteration_index"` - HttpProxy string `json:"http_proxy"` - HttpsProxy string `json:"https_proxy"` + HTTPProxy string `json:"http_proxy"` + HTTPSProxy string `json:"https_proxy"` NoProxy string `json:"no_proxy"` KubernetesConfig string `json:"kubernetes_config,omitempty"` RuntimeConfig string `json:"runtime_config,omitempty"` @@ -20,7 +35,7 @@ type DriverPluginArgs struct { Task string `json:"task"` Type string `json:"type"` CacheDisabledFlag bool `json:"cache_disabled"` - ExecutionIdPath string `json:"execution_id_path"` + ExecutionIDPath string `json:"execution_id_path"` IterationCountPath string `json:"iteration_count_path"` ConditionPath string `json:"condition_path"` PodSpecPathPath string `json:"pod_spec_patch_path"` diff --git a/backend/src/driver/api/response.go b/backend/src/driver/api/response.go index fb571609522..d2f63e9b9a8 100644 --- a/backend/src/driver/api/response.go +++ b/backend/src/driver/api/response.go @@ -1,3 +1,17 @@ +// Copyright 2025 The Kubeflow Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package api type DriverResponse struct { diff --git a/backend/src/driver/execution_paths.go b/backend/src/driver/execution_paths.go index 584d29065d5..20dd6bb1d9e 100644 --- a/backend/src/driver/execution_paths.go +++ b/backend/src/driver/execution_paths.go @@ -1,3 +1,17 @@ +// Copyright 2025 The Kubeflow Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package main type ExecutionPaths struct { diff --git a/backend/src/driver/main.go b/backend/src/driver/main.go index 1ff38f3bf31..b7593441994 100644 --- a/backend/src/driver/main.go +++ b/backend/src/driver/main.go @@ -11,6 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + package main import ( @@ -37,7 +38,7 @@ import ( const ( unsetProxyArgValue = "unset" - ROOT_DAG = "ROOT_DAG" + RootDag = "ROOT_DAG" DAG = "DAG" CONTAINER = "CONTAINER" ) @@ -76,13 +77,13 @@ func init() { flag.Set("stderrthreshold", "WARNING") } -func parseExecConfigJson(k8sExecConfigJson *string) (*kubernetesplatform.KubernetesExecutorConfig, error) { +func parseExecConfigJSON(k8sExecConfigJSON *string) (*kubernetesplatform.KubernetesExecutorConfig, error) { var k8sExecCfg *kubernetesplatform.KubernetesExecutorConfig - if *k8sExecConfigJson != "" { - glog.Infof("input kubernetesConfig:%s\n", prettyPrint(*k8sExecConfigJson)) + if *k8sExecConfigJSON != "" { + glog.Infof("input kubernetesConfig:%s\n", prettyPrint(*k8sExecConfigJSON)) k8sExecCfg = &kubernetesplatform.KubernetesExecutorConfig{} - if err := util.UnmarshalString(*k8sExecConfigJson, k8sExecCfg); err != nil { - return nil, fmt.Errorf("failed to unmarshal Kubernetes config, error: %w\nKubernetesConfig: %v", err, k8sExecConfigJson) + if err := util.UnmarshalString(*k8sExecConfigJSON, k8sExecCfg); err != nil { + return nil, fmt.Errorf("failed to unmarshal Kubernetes config, error: %w\nKubernetesConfig: %v", err, k8sExecConfigJSON) } } return k8sExecCfg, nil @@ -102,7 +103,7 @@ func handleExecution(execution *driver.Execution, driverType string, executionPa return fmt.Errorf("failed to write iteration count to file: %w", err) } } else { - if driverType == ROOT_DAG { + if driverType == RootDag { if err := writeFile(executionPaths.IterationCount, []byte("0")); err != nil { return fmt.Errorf("failed to write iteration count to file: %w", err) } @@ -119,7 +120,7 @@ func handleExecution(execution *driver.Execution, driverType string, executionPa } } else { // nil is a valid value for Condition - if driverType == ROOT_DAG || driverType == CONTAINER { + if driverType == RootDag || driverType == CONTAINER { if err := writeFile(executionPaths.Condition, []byte("nil")); err != nil { return fmt.Errorf("failed to write condition to file: %w", err) } diff --git a/backend/src/driver/main_test.go b/backend/src/driver/main_test.go index ad15a7b00e9..23355c0c94b 100644 --- a/backend/src/driver/main_test.go +++ b/backend/src/driver/main_test.go @@ -1,3 +1,17 @@ +// Copyright 2025 The Kubeflow Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package main import ( @@ -45,7 +59,7 @@ func TestSpecParsing(t *testing.T) { for _, tc := range tt { t.Logf("Running test case: %s", tc.name) - cfg, err := parseExecConfigJson(tc.input) + cfg, err := parseExecConfigJSON(tc.input) assert.Equal(t, tc.wantErr, err != nil) assert.True(t, proto.Equal(tc.expected, cfg)) } @@ -77,7 +91,7 @@ func Test_handleExecutionRootDAG(t *testing.T) { Condition: "condition.txt", } - err := handleExecution(execution, ROOT_DAG, executionPaths) + err := handleExecution(execution, RootDag, executionPaths) if err != nil { t.Errorf("Unexpected error: %v", err) diff --git a/backend/src/driver/rpc_handler.go b/backend/src/driver/rpc_handler.go index db119ae8d56..749e20930af 100644 --- a/backend/src/driver/rpc_handler.go +++ b/backend/src/driver/rpc_handler.go @@ -1,3 +1,17 @@ +// Copyright 2025 The Kubeflow Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package main import ( @@ -94,13 +108,14 @@ func parseDriverRequestArgs(r *http.Request) (*api.DriverPluginArgs, error) { if err := json.NewDecoder(r.Body).Decode(&body); err != nil { return nil, fmt.Errorf("failed to parse driver request body: %v", err) } - if body.Template == nil { + switch { + case body.Template == nil: return nil, fmt.Errorf("driver request body.Template is empty") - } else if body.Template.Plugin == nil { + case body.Template.Plugin == nil: return nil, fmt.Errorf("driver request body.Template.Plugin is empty") - } else if body.Template.Plugin.DriverPlugin == nil { + case body.Template.Plugin.DriverPlugin == nil: return nil, fmt.Errorf("driver request body.Template.Plugin.DriverPlugin is empty") - } else if body.Template.Plugin.DriverPlugin.Args == nil { + case body.Template.Plugin.DriverPlugin.Args == nil: return nil, fmt.Errorf("driver request body.Template.Plugin.Args is empty") } args := body.Template.Plugin.DriverPlugin.Args @@ -118,7 +133,7 @@ func drive(args api.DriverPluginArgs) (execution *driver.Execution, err error) { }() ctx := context.Background() - proxy.InitializeConfig(args.HttpProxy, args.HttpsProxy, args.NoProxy) + proxy.InitializeConfig(args.HTTPProxy, args.HTTPSProxy, args.NoProxy) glog.Infof("input ComponentSpec:%s\n", prettyPrint(args.Component)) componentSpec := &pipelinespec.ComponentSpec{} @@ -149,7 +164,7 @@ func drive(args api.DriverPluginArgs) (execution *driver.Execution, err error) { return nil, fmt.Errorf("failed to unmarshal runtime config, error: %w\nruntimeConfig: %v", err, args.RuntimeConfig) } } - k8sExecCfg, err := parseExecConfigJson(&args.KubernetesConfig) + k8sExecCfg, err := parseExecConfigJSON(&args.KubernetesConfig) if err != nil { return nil, err } @@ -160,7 +175,9 @@ func drive(args api.DriverPluginArgs) (execution *driver.Execution, err error) { var tlsCfg *tls.Config if args.MetadataTLSEnabled { tlsCfg, err = util.GetTLSConfig(args.CACertPath) - return nil, fmt.Errorf("unable to drive driver: failed to load TLS configuration: %v", err) + if err != nil { + return nil, fmt.Errorf("unable to drive driver: failed to load TLS configuration: %v", err) + } } client, err := newMlmdClient(tlsCfg) if err != nil { @@ -180,23 +197,28 @@ func drive(args api.DriverPluginArgs) (execution *driver.Execution, err error) { return nil, fmt.Errorf("failed to parse iteration index, error: %w", err) } options := driver.Options{ - PipelineName: args.PipelineName, - RunID: args.RunID, - RunName: args.RunName, - RunDisplayName: args.RunDisplayName, - Namespace: namespace, - Component: componentSpec, - Task: taskSpec, - DAGExecutionID: dagExecutionID, - IterationIndex: iterationIndex, - PublishLogs: args.PublishLogs, - CacheDisabled: args.CacheDisabledFlag, - DriverType: args.Type, - TaskName: args.TaskName, + PipelineName: args.PipelineName, + RunID: args.RunID, + RunName: args.RunName, + RunDisplayName: args.RunDisplayName, + Namespace: namespace, + Component: componentSpec, + Task: taskSpec, + DAGExecutionID: dagExecutionID, + IterationIndex: iterationIndex, + PublishLogs: args.PublishLogs, + CacheDisabled: args.CacheDisabledFlag, + DriverType: args.Type, + TaskName: args.TaskName, + MLPipelineTLSEnabled: args.MlPipelineTLSEnabled, + MLMDServerAddress: *mlmdServerAddress, + MLMDServerPort: *mlmdServerPort, + CaCertPath: args.CACertPath, } + var driverErr error switch args.Type { - case ROOT_DAG: + case RootDag: options.RuntimeConfig = runtimeConfig execution, driverErr = driver.RootDAG(ctx, options, client) case DAG: @@ -224,19 +246,16 @@ func drive(args api.DriverPluginArgs) (execution *driver.Execution, err error) { } func validate(args api.DriverPluginArgs) error { - if args.Type == "" { + switch { + case args.Type == "": return fmt.Errorf("argument type must be specified") - } - if args.HttpProxy == unsetProxyArgValue { + case args.HTTPProxy == unsetProxyArgValue: return fmt.Errorf("argument http_proxy is required but can be an empty value") - } - if args.HttpsProxy == unsetProxyArgValue { + case args.HTTPSProxy == unsetProxyArgValue: return fmt.Errorf("argument https_proxy is required but can be an empty value") - } - if args.NoProxy == unsetProxyArgValue { + case args.NoProxy == unsetProxyArgValue: return fmt.Errorf("argument no_proxy is required but can be an empty value") } - // validation responsibility lives in driver itself, so we do not validate all other args return nil } @@ -251,18 +270,17 @@ func extractOutputParameters(execution *driver.Execution, driverType string) []a Value: fmt.Sprint(execution.ID), }) } - if execution.IterationCount != nil { + switch { + case execution.IterationCount != nil: outputs = append(outputs, api.Parameter{ Name: "iteration-count", - Value: fmt.Sprint(execution.IterationCount), + Value: fmt.Sprint(*execution.IterationCount), + }) + case driverType == RootDag: + outputs = append(outputs, api.Parameter{ + Name: "iteration-count", + Value: "0", }) - } else { - if driverType == ROOT_DAG { - outputs = append(outputs, api.Parameter{ - Name: "iteration-count", - Value: fmt.Sprint(0), - }) - } } if execution.Cached != nil { outputs = append(outputs, api.Parameter{ diff --git a/backend/src/v2/compiler/argocompiler/container.go b/backend/src/v2/compiler/argocompiler/container.go index bd70cbf01fc..94e253703fa 100644 --- a/backend/src/v2/compiler/argocompiler/container.go +++ b/backend/src/v2/compiler/argocompiler/container.go @@ -105,22 +105,6 @@ func GetLauncherImage() string { return launcherImage } -func GetDriverImage() string { - driverImage := os.Getenv(DriverImageEnvVar) - if driverImage == "" { - driverImage = DefaultDriverImage - } - return driverImage -} - -func GetDriverCommand() []string { - driverCommand := os.Getenv(DriverCommandEnvVar) - if driverCommand == "" { - driverCommand = DefaultDriverCommand - } - return strings.Split(driverCommand, " ") -} - func GetLauncherCommand() []string { launcherCommand := os.Getenv(LauncherCommandEnvVar) if launcherCommand == "" { @@ -148,10 +132,14 @@ func GetPipelineRunAsUser() *int64 { return &runAsUser } -func (c *workflowCompiler) containerDriverTask(name string, inputs containerDriverInputs) (*wfapi.DAGTask, *containerDriverOutputs) { +func (c *workflowCompiler) containerDriverTask(name string, inputs containerDriverInputs) (*wfapi.DAGTask, *containerDriverOutputs, error) { + template, err := c.addContainerDriverTemplate() + if err != nil { + return nil, nil, err + } dagTask := &wfapi.DAGTask{ Name: name, - Template: c.addContainerDriverTemplate(), + Template: template, Arguments: wfapi.Arguments{ Parameters: []wfapi.Parameter{ {Name: paramComponent, Value: wfapi.AnyStringPtr(inputs.component)}, @@ -179,62 +167,61 @@ func (c *workflowCompiler) containerDriverTask(name string, inputs containerDriv cached: taskOutputParameter(name, paramCachedDecision), condition: taskOutputParameter(name, paramCondition), } - return dagTask, outputs + return dagTask, outputs, nil } -func (c *workflowCompiler) addContainerDriverTemplate() string { +func (c *workflowCompiler) addContainerDriverTemplate() (string, error) { name := "system-container-driver" _, ok := c.templates[name] if ok { - return name - } - - args := []string{ - "--type", "CONTAINER", - "--pipeline_name", c.spec.GetPipelineInfo().GetName(), - "--run_id", runID(), - "--run_name", runResourceName(), - "--run_display_name", c.job.DisplayName, - "--dag_execution_id", inputValue(paramParentDagID), - "--component", inputValue(paramComponent), - "--task", inputValue(paramTask), - "--task_name", inputValue(paramTaskName), - "--container", inputValue(paramContainer), - "--iteration_index", inputValue(paramIterationIndex), - "--cached_decision_path", outputPath(paramCachedDecision), - "--pod_spec_patch_path", outputPath(paramPodSpecPatch), - "--condition_path", outputPath(paramCondition), - "--kubernetes_config", inputValue(paramKubernetesConfig), - "--http_proxy", proxy.GetConfig().GetHttpProxy(), - "--https_proxy", proxy.GetConfig().GetHttpsProxy(), - "--no_proxy", proxy.GetConfig().GetNoProxy(), - "--ml_pipeline_server_address", config.GetMLPipelineServerConfig().Address, - "--ml_pipeline_server_port", config.GetMLPipelineServerConfig().Port, - "--mlmd_server_address", metadata.GetMetadataConfig().Address, - "--mlmd_server_port", metadata.GetMetadataConfig().Port, - } - if c.cacheDisabled { - args = append(args, "--cache_disabled") - } - if c.mlPipelineTLSEnabled { - args = append(args, "--ml_pipeline_tls_enabled") - } - if common.GetMetadataTLSEnabled() { - args = append(args, "--metadata_tls_enabled") + return name, nil + } + + args := map[string]interface{}{ + "type": "CONTAINER", + "pipeline_name": c.spec.GetPipelineInfo().GetName(), + "run_id": runID(), + "run_name": runResourceName(), + "run_display_name": c.job.DisplayName, + "dag_execution_id": inputValue(paramParentDagID), + "component": inputValue(paramComponent), + "task": inputValue(paramTask), + "task_name": inputValue(paramTaskName), + "container": inputValue(paramContainer), + "iteration_index": inputValue(paramIterationIndex), + "cached_decision_path": outputPath(paramCachedDecision), + "pod_spec_patch_path": outputPath(paramPodSpecPatch), + "condition_path": outputPath(paramCondition), + "kubernetes_config": inputValue(paramKubernetesConfig), + "http_proxy": proxy.GetConfig().GetHttpProxy(), + "https_proxy": proxy.GetConfig().GetHttpsProxy(), + "no_proxy": proxy.GetConfig().GetNoProxy(), + "ml_pipeline_server_address", config.GetMLPipelineServerConfig().Address, + "ml_pipeline_server_port", config.GetMLPipelineServerConfig().Port, + "mlmd_server_address": metadata.DefaultConfig().Address, + "mlmd_server_port": metadata.DefaultConfig().Port, + "cache_disabled": c.cacheDisabled, + "ml_pipeline_tls_enabled": c.mlPipelineTLSEnabled, + "metadata_tls_enabled": common.GetMetadataTLSEnabled(), } setCABundle := false // If CABUNDLE_SECRET_NAME or CABUNDLE_CONFIGMAP_NAME is set, add ca_cert_path arg to container driver. if common.GetCaBundleSecretName() != "" || common.GetCaBundleConfigMapName() != "" { - args = append(args, "--ca_cert_path", common.CustomCaCertPath) + args["ca_cert_path"] = common.CustomCaCertPath setCABundle = true } if value, ok := os.LookupEnv(PipelineLogLevelEnvVar); ok { - args = append(args, "--log_level", value) + args["log_level"] = value } if value, ok := os.LookupEnv(PublishLogsEnvVar); ok { - args = append(args, "--publish_logs", value) + args["publish_logs"] = value + } + + containerDriverPlugin, err := driverPlugin(args) + if err != nil { + return name, fmt.Errorf("failed to add container driver plugin: %v", err) } template := &wfapi.Template{ @@ -252,18 +239,12 @@ func (c *workflowCompiler) addContainerDriverTemplate() string { }, Outputs: wfapi.Outputs{ Parameters: []wfapi.Parameter{ - {Name: paramPodSpecPatch, ValueFrom: &wfapi.ValueFrom{Path: "/tmp/outputs/pod-spec-patch", Default: wfapi.AnyStringPtr("")}}, - {Name: paramCachedDecision, Default: wfapi.AnyStringPtr("false"), ValueFrom: &wfapi.ValueFrom{Path: "/tmp/outputs/cached-decision", Default: wfapi.AnyStringPtr("false")}}, - {Name: paramCondition, ValueFrom: &wfapi.ValueFrom{Path: "/tmp/outputs/condition", Default: wfapi.AnyStringPtr("true")}}, + {Name: paramPodSpecPatch, ValueFrom: &wfapi.ValueFrom{JSONPath: "$.pod-spec-patch", Default: wfapi.AnyStringPtr("")}}, + {Name: paramCachedDecision, Default: wfapi.AnyStringPtr("false"), ValueFrom: &wfapi.ValueFrom{JSONPath: "$.cached-decision", Default: wfapi.AnyStringPtr("false")}}, + {Name: paramCondition, ValueFrom: &wfapi.ValueFrom{JSONPath: "$.condition", Default: wfapi.AnyStringPtr("true")}}, }, }, - Container: &k8score.Container{ - Image: c.driverImage, - Command: c.driverCommand, - Args: args, - Resources: driverResources, - Env: append(proxy.GetConfig().GetEnvVars(), commonEnvs...), - }, + Plugin: containerDriverPlugin, } applySecurityContextToTemplate(template) // If TLS is enabled (apiserver or metadata), add the custom CA bundle to the container driver template. @@ -272,7 +253,7 @@ func (c *workflowCompiler) addContainerDriverTemplate() string { } c.templates[name] = template c.wf.Spec.Templates = append(c.wf.Spec.Templates, *template) - return name + return name, err } type containerExecutorInputs struct { diff --git a/backend/src/v2/compiler/argocompiler/dag.go b/backend/src/v2/compiler/argocompiler/dag.go index beecbb70dc3..db0c3e9f606 100644 --- a/backend/src/v2/compiler/argocompiler/dag.go +++ b/backend/src/v2/compiler/argocompiler/dag.go @@ -27,7 +27,6 @@ import ( "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" "github.com/kubeflow/pipelines/backend/src/apiserver/common" "github.com/kubeflow/pipelines/backend/src/v2/compiler" - k8score "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/intstr" ) @@ -286,7 +285,7 @@ func (c *workflowCompiler) task(name string, task *pipelinespec.PipelineTaskSpec driverTaskName := name + "-driver" // The following call will return an empty string for tasks without kubernetes-specific annotation. kubernetesConfigPlaceholder, _ := c.useKubernetesImpl(componentName) - driver, driverOutputs := c.containerDriverTask(driverTaskName, containerDriverInputs{ + driver, driverOutputs, err := c.containerDriverTask(driverTaskName, containerDriverInputs{ component: componentSpecPlaceholder, task: taskSpecJson, container: containerPlaceholder, @@ -295,6 +294,9 @@ func (c *workflowCompiler) task(name string, task *pipelinespec.PipelineTaskSpec kubernetesConfig: kubernetesConfigPlaceholder, taskName: name, }) + if err != nil { + return nil, err + } if task.GetTriggerPolicy().GetCondition() == "" { driverOutputs.condition = "" } @@ -534,9 +536,13 @@ func (c *workflowCompiler) dagDriverTask(name string, inputs dagDriverInputs) (* Value: wfapi.AnyStringPtr(inputs.taskName), }) } + dagTemplate, err := c.addDAGDriverTemplate() + if err != nil { + return nil, nil, err + } t := &wfapi.DAGTask{ Name: name, - Template: c.addDAGDriverTemplate(), + Template: dagTemplate, Arguments: wfapi.Arguments{ Parameters: params, }, @@ -548,58 +554,57 @@ func (c *workflowCompiler) dagDriverTask(name string, inputs dagDriverInputs) (* }, nil } -func (c *workflowCompiler) addDAGDriverTemplate() string { +func (c *workflowCompiler) addDAGDriverTemplate() (string, error) { name := "system-dag-driver" _, ok := c.templates[name] if ok { - return name - } - - args := []string{ - "--type", inputValue(paramDriverType), - "--pipeline_name", c.spec.GetPipelineInfo().GetName(), - "--run_id", runID(), - "--run_name", runResourceName(), - "--run_display_name", c.job.DisplayName, - "--dag_execution_id", inputValue(paramParentDagID), - "--component", inputValue(paramComponent), - "--task", inputValue(paramTask), - "--task_name", inputValue(paramTaskName), - "--runtime_config", inputValue(paramRuntimeConfig), - "--iteration_index", inputValue(paramIterationIndex), - "--execution_id_path", outputPath(paramExecutionID), - "--iteration_count_path", outputPath(paramIterationCount), - "--condition_path", outputPath(paramCondition), - "--http_proxy", proxy.GetConfig().GetHttpProxy(), - "--https_proxy", proxy.GetConfig().GetHttpsProxy(), - "--no_proxy", proxy.GetConfig().GetNoProxy(), - "--ml_pipeline_server_address", config.GetMLPipelineServerConfig().Address, - "--ml_pipeline_server_port", config.GetMLPipelineServerConfig().Port, - "--mlmd_server_address", metadata.GetMetadataConfig().Address, - "--mlmd_server_port", metadata.GetMetadataConfig().Port, - } - if c.cacheDisabled { - args = append(args, "--cache_disabled") - } - if c.mlPipelineTLSEnabled { - args = append(args, "--ml_pipeline_tls_enabled") - } - if common.GetMetadataTLSEnabled() { - args = append(args, "--metadata_tls_enabled") + return name, nil + } + + args := map[string]interface{}{ + "type": inputValue(paramDriverType), + "pipeline_name": c.spec.GetPipelineInfo().GetName(), + "run_id": runID(), + "run_name": runResourceName(), + "run_display_name": c.job.DisplayName, + "dag_execution_id": inputValue(paramParentDagID), + "component": inputValue(paramComponent), + "task": inputValue(paramTask), + "task_name": inputValue(paramTaskName), + "runtime_config": inputValue(paramRuntimeConfig), + "iteration_index": inputValue(paramIterationIndex), + "execution_id_path": outputPath(paramExecutionID), + "iteration_count_path": outputPath(paramIterationCount), + "condition_path": outputPath(paramCondition), + "http_proxy": proxy.GetConfig().GetHttpProxy(), + "https_proxy": proxy.GetConfig().GetHttpsProxy(), + "no_proxy": proxy.GetConfig().GetNoProxy(), + "ml_pipeline_server_address", config.GetMLPipelineServerConfig().Address, + "ml_pipeline_server_port", config.GetMLPipelineServerConfig().Port, + "mlmd_server_address": metadata.DefaultConfig().Address, + "mlmd_server_port": metadata.DefaultConfig().Port, + "cache_disabled": c.cacheDisabled, + "ml_pipeline_tls_enabled": c.mlPipelineTLSEnabled, + "metadata_tls_enabled": common.GetMetadataTLSEnabled(), } setCABundle := false // If CABUNDLE_SECRET_NAME or CABUNDLE_CONFIGMAP_NAME is set, add ca_cert_path arg to DAG driver. if common.GetCaBundleSecretName() != "" || common.GetCaBundleConfigMapName() != "" { - args = append(args, "--ca_cert_path", common.CustomCaCertPath) + args["ca_cert_path"] = common.CustomCaCertPath setCABundle = true } if value, ok := os.LookupEnv(PipelineLogLevelEnvVar); ok { - args = append(args, "--log_level", value) + args["log_level"] = value } if value, ok := os.LookupEnv(PublishLogsEnvVar); ok { - args = append(args, "--publish_logs", value) + args["publish_logs"] = value + } + + dagPlugin, err := driverPlugin(args) + if err != nil { + return "", err } template := &wfapi.Template{ @@ -622,13 +627,7 @@ func (c *workflowCompiler) addDAGDriverTemplate() string { {Name: paramCondition, ValueFrom: &wfapi.ValueFrom{Path: "/tmp/outputs/condition", Default: wfapi.AnyStringPtr("true")}}, }, }, - Container: &k8score.Container{ - Image: c.driverImage, - Command: c.driverCommand, - Args: args, - Resources: driverResources, - Env: proxy.GetConfig().GetEnvVars(), - }, + Plugin: dagPlugin, } applySecurityContextToTemplate(template) // If TLS is enabled (apiserver or metadata), add the custom CA bundle to the DAG driver template. @@ -637,7 +636,7 @@ func (c *workflowCompiler) addDAGDriverTemplate() string { } c.templates[name] = template c.wf.Spec.Templates = append(c.wf.Spec.Templates, *template) - return name + return name, nil } func addImplicitDependencies(dagSpec *pipelinespec.DagSpec) error { diff --git a/backend/src/v2/compiler/argocompiler/plugin.go b/backend/src/v2/compiler/argocompiler/plugin.go index 941f9fcd04b..a202c40270a 100644 --- a/backend/src/v2/compiler/argocompiler/plugin.go +++ b/backend/src/v2/compiler/argocompiler/plugin.go @@ -1,8 +1,23 @@ +// Copyright 2025 The Kubeflow Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package argocompiler import ( "encoding/json" "fmt" + wfapi "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" ) diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml index 6467e3cb988..98f9552b8b1 100644 --- a/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml +++ b/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml @@ -132,7 +132,7 @@ spec: # JSON patch to apply to compiled workflow specifications - name: COMPILED_PIPELINE_SPEC_PATCH value: "{}" - image: ntny/kfp-api-server:central-driver-beta + image: ntny/kfp-api-server:beta-poc imagePullPolicy: IfNotPresent name: ml-pipeline-api-server ports: diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml index 66cfc5caabd..d116a40b47c 100644 --- a/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml +++ b/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml @@ -7,7 +7,7 @@ metadata: data: sidecar.automountServiceAccountToken: "true" sidecar.container: | - image: ntny/kfp-driver:central-driver-beta + image: ntny/kfp-driver:beta-poc name: driver-plugin ports: - containerPort: 8080 diff --git a/manifests/kustomize/env/dev/kustomization.yaml b/manifests/kustomize/env/dev/kustomization.yaml index 72b7adbbf33..b9dde28236f 100644 --- a/manifests/kustomize/env/dev/kustomization.yaml +++ b/manifests/kustomize/env/dev/kustomization.yaml @@ -9,8 +9,8 @@ resources: - ../gcp/inverse-proxy images: -- name: ntny/kfp-driver - newTag: central-driver-poc +- name: ghcr.io/kubeflow/kfp-driver + newTag: master - name: ghcr.io/kubeflow/kfp-frontend newTag: master - name: ghcr.io/kubeflow/kfp-persistence-agent diff --git a/test_data/compiled-workflows/arguments-parameters.yaml b/test_data/compiled-workflows/arguments-parameters.yaml deleted file mode 100644 index 5d6a324eff3..00000000000 --- a/test_data/compiled-workflows/arguments-parameters.yaml +++ /dev/null @@ -1,334 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - creationTimestamp: null - generateName: echo- -spec: - arguments: - parameters: - - name: components-e3bf4dafebca73c53759f2310029cb3fc65ab6a05d870069f7c58096ff7bb483 - value: '{"executorLabel":"exec-echo","inputDefinitions":{"parameters":{"param1":{"parameterType":"STRING"},"param2":{"parameterType":"STRING"}}}}' - - name: implementations-e3bf4dafebca73c53759f2310029cb3fc65ab6a05d870069f7c58096ff7bb483 - value: '{"args":["{{$.inputs.parameters[''param1'']}}-{{$.inputs.parameters[''param2'']}}"],"command":["echo"],"image":"public.ecr.aws/docker/library/python:3.12"}' - - name: components-root - value: '{"dag":{"tasks":{"echo":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-echo"},"inputs":{"parameters":{"param1":{"componentInputParameter":"param1"},"param2":{"componentInputParameter":"param2"}}},"taskInfo":{"name":"echo"}}}},"inputDefinitions":{"parameters":{"param1":{"defaultValue":"hello","parameterType":"STRING"},"param2":{"parameterType":"STRING"}}}}' - entrypoint: entrypoint - podMetadata: - annotations: - pipelines.kubeflow.org/v2_component: "true" - labels: - pipelines.kubeflow.org/v2_component: "true" - serviceAccountName: pipeline-runner - templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: - parameters: - - name: component - - name: task - - name: container - - name: task-name - - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: "" - name: kubernetes-config - metadata: {} - name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - name: executor - template: system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: - - name: pod-spec-patch - - default: "false" - name: cached-decision - metadata: {} - name: system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher:latest - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 100m - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - metadata: {} - name: system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-e3bf4dafebca73c53759f2310029cb3fc65ab6a05d870069f7c58096ff7bb483}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-echo"},"inputs":{"parameters":{"param1":{"componentInputParameter":"param1"},"param2":{"componentInputParameter":"param2"}}},"taskInfo":{"name":"echo"}}' - - name: container - value: '{{workflow.parameters.implementations-e3bf4dafebca73c53759f2310029cb3fc65ab6a05d870069f7c58096ff7bb483}}' - - name: task-name - value: echo - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: echo-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.echo-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.echo-driver.outputs.parameters.cached-decision}}' - depends: echo-driver.Succeeded - name: echo - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: root - outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: - parameters: - - name: component - - default: "" - name: runtime-config - - default: "" - name: task - - default: "" - name: task-name - - default: "0" - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: DAG - name: driver-type - metadata: {} - name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-root}}' - - name: runtime-config - value: '{"parameterValues":{"param1":"hello"}}' - - name: driver-type - value: ROOT_DAG - name: root-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' - - name: condition - value: "" - depends: root-driver.Succeeded - name: root - template: root - inputs: {} - metadata: {} - name: entrypoint - outputs: {} -status: - finishedAt: null - startedAt: null diff --git a/test_data/compiled-workflows/arguments.pipeline.yaml b/test_data/compiled-workflows/arguments.pipeline.yaml deleted file mode 100644 index 5d6a324eff3..00000000000 --- a/test_data/compiled-workflows/arguments.pipeline.yaml +++ /dev/null @@ -1,334 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - creationTimestamp: null - generateName: echo- -spec: - arguments: - parameters: - - name: components-e3bf4dafebca73c53759f2310029cb3fc65ab6a05d870069f7c58096ff7bb483 - value: '{"executorLabel":"exec-echo","inputDefinitions":{"parameters":{"param1":{"parameterType":"STRING"},"param2":{"parameterType":"STRING"}}}}' - - name: implementations-e3bf4dafebca73c53759f2310029cb3fc65ab6a05d870069f7c58096ff7bb483 - value: '{"args":["{{$.inputs.parameters[''param1'']}}-{{$.inputs.parameters[''param2'']}}"],"command":["echo"],"image":"public.ecr.aws/docker/library/python:3.12"}' - - name: components-root - value: '{"dag":{"tasks":{"echo":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-echo"},"inputs":{"parameters":{"param1":{"componentInputParameter":"param1"},"param2":{"componentInputParameter":"param2"}}},"taskInfo":{"name":"echo"}}}},"inputDefinitions":{"parameters":{"param1":{"defaultValue":"hello","parameterType":"STRING"},"param2":{"parameterType":"STRING"}}}}' - entrypoint: entrypoint - podMetadata: - annotations: - pipelines.kubeflow.org/v2_component: "true" - labels: - pipelines.kubeflow.org/v2_component: "true" - serviceAccountName: pipeline-runner - templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: - parameters: - - name: component - - name: task - - name: container - - name: task-name - - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: "" - name: kubernetes-config - metadata: {} - name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - name: executor - template: system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: - - name: pod-spec-patch - - default: "false" - name: cached-decision - metadata: {} - name: system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher:latest - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 100m - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - metadata: {} - name: system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-e3bf4dafebca73c53759f2310029cb3fc65ab6a05d870069f7c58096ff7bb483}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-echo"},"inputs":{"parameters":{"param1":{"componentInputParameter":"param1"},"param2":{"componentInputParameter":"param2"}}},"taskInfo":{"name":"echo"}}' - - name: container - value: '{{workflow.parameters.implementations-e3bf4dafebca73c53759f2310029cb3fc65ab6a05d870069f7c58096ff7bb483}}' - - name: task-name - value: echo - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: echo-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.echo-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.echo-driver.outputs.parameters.cached-decision}}' - depends: echo-driver.Succeeded - name: echo - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: root - outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: - parameters: - - name: component - - default: "" - name: runtime-config - - default: "" - name: task - - default: "" - name: task-name - - default: "0" - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: DAG - name: driver-type - metadata: {} - name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-root}}' - - name: runtime-config - value: '{"parameterValues":{"param1":"hello"}}' - - name: driver-type - value: ROOT_DAG - name: root-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' - - name: condition - value: "" - depends: root-driver.Succeeded - name: root - template: root - inputs: {} - metadata: {} - name: entrypoint - outputs: {} -status: - finishedAt: null - startedAt: null diff --git a/test_data/compiled-workflows/hello-world.yaml b/test_data/compiled-workflows/hello-world.yaml deleted file mode 100644 index c86947cdf41..00000000000 --- a/test_data/compiled-workflows/hello-world.yaml +++ /dev/null @@ -1,334 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - creationTimestamp: null - generateName: echo- -spec: - arguments: - parameters: - - name: components-cf9c81ac9e6ab0dcdd92cb89ed717317e681cb0645cb5ddfc4824b1de14346b3 - value: '{"executorLabel":"exec-echo"}' - - name: implementations-cf9c81ac9e6ab0dcdd92cb89ed717317e681cb0645cb5ddfc4824b1de14346b3 - value: '{"args":["hello world"],"command":["echo"],"image":"public.ecr.aws/docker/library/python:3.12"}' - - name: components-root - value: '{"dag":{"tasks":{"echo":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-echo"},"taskInfo":{"name":"echo"}}}}}' - entrypoint: entrypoint - podMetadata: - annotations: - pipelines.kubeflow.org/v2_component: "true" - labels: - pipelines.kubeflow.org/v2_component: "true" - serviceAccountName: pipeline-runner - templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: - parameters: - - name: component - - name: task - - name: container - - name: task-name - - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: "" - name: kubernetes-config - metadata: {} - name: system-container-driver - outputs: - parameters: - - name: pod-spec-patch - valueFrom: - default: "" - path: /tmp/outputs/pod-spec-patch - - default: "false" - name: cached-decision - valueFrom: - default: "false" - path: /tmp/outputs/cached-decision - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - - dag: - tasks: - - arguments: - parameters: - - name: pod-spec-patch - value: '{{inputs.parameters.pod-spec-patch}}' - name: executor - template: system-container-impl - when: '{{inputs.parameters.cached-decision}} != true' - inputs: - parameters: - - name: pod-spec-patch - - default: "false" - name: cached-decision - metadata: {} - name: system-container-executor - outputs: {} - - container: - command: - - should-be-overridden-during-runtime - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - envFrom: - - configMapRef: - name: metadata-grpc-configmap - optional: true - image: gcr.io/ml-pipeline/should-be-overridden-during-runtime - name: "" - resources: {} - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - - mountPath: /gcs - name: gcs-scratch - - mountPath: /s3 - name: s3-scratch - - mountPath: /minio - name: minio-scratch - - mountPath: /.local - name: dot-local-scratch - - mountPath: /.cache - name: dot-cache-scratch - - mountPath: /.config - name: dot-config-scratch - initContainers: - - args: - - --copy - - /kfp-launcher/launch - command: - - launcher-v2 - image: ghcr.io/kubeflow/kfp-launcher:latest - name: kfp-launcher - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 100m - volumeMounts: - - mountPath: /kfp-launcher - name: kfp-launcher - inputs: - parameters: - - name: pod-spec-patch - metadata: {} - name: system-container-impl - outputs: {} - podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' - volumes: - - emptyDir: {} - name: kfp-launcher - - emptyDir: {} - name: gcs-scratch - - emptyDir: {} - name: s3-scratch - - emptyDir: {} - name: minio-scratch - - emptyDir: {} - name: dot-local-scratch - - emptyDir: {} - name: dot-cache-scratch - - emptyDir: {} - name: dot-config-scratch - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-cf9c81ac9e6ab0dcdd92cb89ed717317e681cb0645cb5ddfc4824b1de14346b3}}' - - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-echo"},"taskInfo":{"name":"echo"}}' - - name: container - value: '{{workflow.parameters.implementations-cf9c81ac9e6ab0dcdd92cb89ed717317e681cb0645cb5ddfc4824b1de14346b3}}' - - name: task-name - value: echo - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: echo-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.echo-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.echo-driver.outputs.parameters.cached-decision}}' - depends: echo-driver.Succeeded - name: echo - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: root - outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: - parameters: - - name: component - - default: "" - name: runtime-config - - default: "" - name: task - - default: "" - name: task-name - - default: "0" - name: parent-dag-id - - default: "-1" - name: iteration-index - - default: DAG - name: driver-type - metadata: {} - name: system-dag-driver - outputs: - parameters: - - name: execution-id - valueFrom: - path: /tmp/outputs/execution-id - - name: iteration-count - valueFrom: - default: "0" - path: /tmp/outputs/iteration-count - - name: condition - valueFrom: - default: "true" - path: /tmp/outputs/condition - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-root}}' - - name: runtime-config - value: '{}' - - name: driver-type - value: ROOT_DAG - name: root-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.root-driver.outputs.parameters.execution-id}}' - - name: condition - value: "" - depends: root-driver.Succeeded - name: root - template: root - inputs: {} - metadata: {} - name: entrypoint - outputs: {} -status: - finishedAt: null - startedAt: null From 38cbe1835316841d0eb789cc9f8a6aef3cc4fd39 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Thu, 25 Dec 2025 01:29:34 +0300 Subject: [PATCH 07/51] - update branch Signed-off-by: arpechenin --- backend/src/driver/api/request.go | 55 +-- backend/src/driver/main.go | 9 +- backend/src/driver/rpc_handler.go | 41 +- backend/src/v2/cmd/driver/main.go | 368 ------------------ .../src/v2/compiler/argocompiler/container.go | 51 ++- backend/src/v2/compiler/argocompiler/dag.go | 54 ++- backend/test/compiler/argo_ginkgo_test.go | 22 +- backend/test/end2end/e2e_suite_test.go | 1 - .../ml-pipeline-apiserver-deployment.yaml | 2 +- .../ml-pipeline-driver-plugin-cm.yaml | 2 +- 10 files changed, 118 insertions(+), 487 deletions(-) diff --git a/backend/src/driver/api/request.go b/backend/src/driver/api/request.go index 0da025e4bb4..e2e73d5cb0a 100644 --- a/backend/src/driver/api/request.go +++ b/backend/src/driver/api/request.go @@ -16,32 +16,35 @@ package api type DriverPluginArgs struct { - CachedDecisionPath string `json:"cached_decision_path"` - Component string `json:"component,omitempty"` - Container string `json:"container,omitempty"` - DagExecutionID string `json:"dag_execution_id"` - IterationIndex string `json:"iteration_index"` - HTTPProxy string `json:"http_proxy"` - HTTPSProxy string `json:"https_proxy"` - NoProxy string `json:"no_proxy"` - KubernetesConfig string `json:"kubernetes_config,omitempty"` - RuntimeConfig string `json:"runtime_config,omitempty"` - PipelineName string `json:"pipeline_name"` - PublishLogs string `json:"publish_logs,omitempty"` - RunID string `json:"run_id"` - RunName string `json:"run_name"` - RunDisplayName string `json:"run_display_name"` - TaskName string `json:"task_name"` - Task string `json:"task"` - Type string `json:"type"` - CacheDisabledFlag bool `json:"cache_disabled"` - ExecutionIDPath string `json:"execution_id_path"` - IterationCountPath string `json:"iteration_count_path"` - ConditionPath string `json:"condition_path"` - PodSpecPathPath string `json:"pod_spec_patch_path"` - MlPipelineTLSEnabled bool `json:"ml_pipeline_tls_enabled"` - MetadataTLSEnabled bool `json:"metadata_tls_enabled"` - CACertPath string `json:"ca_cert_path"` + CachedDecisionPath string `json:"cached_decision_path"` + Component string `json:"component,omitempty"` + Container string `json:"container,omitempty"` + DagExecutionID string `json:"dag_execution_id"` + IterationIndex string `json:"iteration_index"` + HTTPProxy string `json:"http_proxy"` + HTTPSProxy string `json:"https_proxy"` + NoProxy string `json:"no_proxy"` + KubernetesConfig string `json:"kubernetes_config,omitempty"` + RuntimeConfig string `json:"runtime_config,omitempty"` + PipelineName string `json:"pipeline_name"` + PublishLogs string `json:"publish_logs,omitempty"` + RunID string `json:"run_id"` + RunName string `json:"run_name"` + RunDisplayName string `json:"run_display_name"` + TaskName string `json:"task_name"` + Task string `json:"task"` + Type string `json:"type"` + CacheDisabledFlag bool `json:"cache_disabled"` + ExecutionIDPath string `json:"execution_id_path"` + IterationCountPath string `json:"iteration_count_path"` + ConditionPath string `json:"condition_path"` + PodSpecPathPath string `json:"pod_spec_patch_path"` + MlPipelineServerAddress string `json:"ml_pipeline_server_address"` + MlPipelineServerPort string `json:"ml_pipeline_server_port"` + MlPipelineTLSEnabled bool `json:"ml_pipeline_tls_enabled"` + MetadataTLSEnabled bool `json:"metadata_tls_enabled"` + CACertPath string `json:"ca_cert_path"` + LogLevel string `json:"log_level"` } type DriverPlugin struct { diff --git a/backend/src/driver/main.go b/backend/src/driver/main.go index b7593441994..c2bd03daddc 100644 --- a/backend/src/driver/main.go +++ b/backend/src/driver/main.go @@ -170,11 +170,6 @@ func writeFile(path string, data []byte) (err error) { return os.WriteFile(path, data, 0o644) } -func newMlmdClient(tlsCfg *tls.Config) (*metadata.Client, error) { - mlmdConfig := metadata.DefaultConfig() - if *mlmdServerAddress != "" && *mlmdServerPort != "" { - mlmdConfig.Address = *mlmdServerAddress - mlmdConfig.Port = *mlmdServerPort - } - return metadata.NewClient(mlmdConfig.Address, mlmdConfig.Port, tlsCfg) +func newMlmdClient(mlmdServerAddress string, mlmdServerPort string, tlsCfg *tls.Config) (*metadata.Client, error) { + return metadata.NewClient(mlmdServerAddress, mlmdServerPort, tlsCfg) } diff --git a/backend/src/driver/rpc_handler.go b/backend/src/driver/rpc_handler.go index 749e20930af..1b08e5d3409 100644 --- a/backend/src/driver/rpc_handler.go +++ b/backend/src/driver/rpc_handler.go @@ -179,11 +179,11 @@ func drive(args api.DriverPluginArgs) (execution *driver.Execution, err error) { return nil, fmt.Errorf("unable to drive driver: failed to load TLS configuration: %v", err) } } - client, err := newMlmdClient(tlsCfg) + client, err := newMlmdClient(*mlmdServerAddress, *mlmdServerPort, tlsCfg) if err != nil { return nil, err } - cacheClient, err := cacheutils.NewClient(args.CacheDisabledFlag, tlsCfg) + cacheClient, err := cacheutils.NewClient(args.MlPipelineServerAddress, args.MlPipelineServerPort, args.CacheDisabledFlag, tlsCfg) if err != nil { return nil, err } @@ -197,23 +197,26 @@ func drive(args api.DriverPluginArgs) (execution *driver.Execution, err error) { return nil, fmt.Errorf("failed to parse iteration index, error: %w", err) } options := driver.Options{ - PipelineName: args.PipelineName, - RunID: args.RunID, - RunName: args.RunName, - RunDisplayName: args.RunDisplayName, - Namespace: namespace, - Component: componentSpec, - Task: taskSpec, - DAGExecutionID: dagExecutionID, - IterationIndex: iterationIndex, - PublishLogs: args.PublishLogs, - CacheDisabled: args.CacheDisabledFlag, - DriverType: args.Type, - TaskName: args.TaskName, - MLPipelineTLSEnabled: args.MlPipelineTLSEnabled, - MLMDServerAddress: *mlmdServerAddress, - MLMDServerPort: *mlmdServerPort, - CaCertPath: args.CACertPath, + PipelineName: args.PipelineName, + RunID: args.RunID, + RunName: args.RunName, + RunDisplayName: args.RunDisplayName, + Namespace: namespace, + Component: componentSpec, + Task: taskSpec, + DAGExecutionID: dagExecutionID, + IterationIndex: iterationIndex, + PipelineLogLevel: args.LogLevel, + PublishLogs: args.PublishLogs, + CacheDisabled: args.CacheDisabledFlag, + DriverType: args.Type, + TaskName: args.TaskName, + MLPipelineServerAddress: args.MlPipelineServerAddress, + MLPipelineServerPort: args.MlPipelineServerPort, + MLPipelineTLSEnabled: args.MlPipelineTLSEnabled, + MLMDServerAddress: *mlmdServerAddress, + MLMDServerPort: *mlmdServerPort, + CaCertPath: args.CACertPath, } var driverErr error diff --git a/backend/src/v2/cmd/driver/main.go b/backend/src/v2/cmd/driver/main.go index b3faccbc489..e69de29bb2d 100644 --- a/backend/src/v2/cmd/driver/main.go +++ b/backend/src/v2/cmd/driver/main.go @@ -1,368 +0,0 @@ -// Copyright 2021-2023 The Kubeflow Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package main - -import ( - "bytes" - "context" - "crypto/tls" - "encoding/json" - "flag" - "fmt" - - "google.golang.org/protobuf/encoding/protojson" - - "github.com/kubeflow/pipelines/backend/src/apiserver/config/proxy" - "github.com/kubeflow/pipelines/backend/src/common/util" - - "os" - "path/filepath" - "strconv" - "strings" - - "github.com/golang/glog" - "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" - "github.com/kubeflow/pipelines/backend/src/v2/cacheutils" - "github.com/kubeflow/pipelines/backend/src/v2/config" - "github.com/kubeflow/pipelines/backend/src/v2/driver" - "github.com/kubeflow/pipelines/backend/src/v2/metadata" - "github.com/kubeflow/pipelines/kubernetes_platform/go/kubernetesplatform" -) - -const ( - driverTypeArg = "type" - httpProxyArg = "http_proxy" - httpsProxyArg = "https_proxy" - noProxyArg = "no_proxy" - unsetProxyArgValue = "unset" - ROOT_DAG = "ROOT_DAG" - DAG = "DAG" - CONTAINER = "CONTAINER" -) - -var ( - // inputs - driverType = flag.String(driverTypeArg, "", "task driver type, one of ROOT_DAG, DAG, CONTAINER") - pipelineName = flag.String("pipeline_name", "", "pipeline context name") - runID = flag.String("run_id", "", "pipeline run uid") - runName = flag.String("run_name", "", "pipeline run name (Kubernetes object name)") - runDisplayName = flag.String("run_display_name", "", "pipeline run display name") - componentSpecJson = flag.String("component", "{}", "component spec") - taskSpecJson = flag.String("task", "", "task spec") - runtimeConfigJson = flag.String("runtime_config", "", "jobruntime config") - iterationIndex = flag.Int("iteration_index", -1, "iteration index, -1 means not an interation") - taskName = flag.String("task_name", "", "original task name, used for proper input resolution in the container/dag driver") - - // container inputs - dagExecutionID = flag.Int64("dag_execution_id", 0, "DAG execution ID") - containerSpecJson = flag.String("container", "{}", "container spec") - k8sExecConfigJson = flag.String("kubernetes_config", "{}", "kubernetes executor config") - - // config - mlPipelineServerAddress = flag.String("ml_pipeline_server_address", "ml-pipeline", "The name of the ML pipeline API server address.") - mlPipelineServerPort = flag.String("ml_pipeline_server_port", "8887", "The port of the ML pipeline API server.") - mlmdServerAddress = flag.String("mlmd_server_address", "", "MLMD server address") - mlmdServerPort = flag.String("mlmd_server_port", "", "MLMD server port") - - // output paths - executionIDPath = flag.String("execution_id_path", "", "Exeucution ID output path") - iterationCountPath = flag.String("iteration_count_path", "", "Iteration Count output path") - podSpecPatchPath = flag.String("pod_spec_patch_path", "", "Pod Spec Patch output path") - // the value stored in the paths will be either 'true' or 'false' - cachedDecisionPath = flag.String("cached_decision_path", "", "Cached Decision output path") - conditionPath = flag.String("condition_path", "", "Condition output path") - logLevel = flag.String("log_level", "1", "The verbosity level to log.") - - // proxy - httpProxy = flag.String(httpProxyArg, unsetProxyArgValue, "The proxy for HTTP connections.") - httpsProxy = flag.String(httpsProxyArg, unsetProxyArgValue, "The proxy for HTTPS connections.") - noProxy = flag.String(noProxyArg, unsetProxyArgValue, "Addresses that should ignore the proxy.") - publishLogs = flag.String("publish_logs", "true", "Whether to publish component logs to the object store") - cacheDisabledFlag = flag.Bool("cache_disabled", false, "Disable cache globally.") - mlPipelineTLSEnabled = flag.Bool("ml_pipeline_tls_enabled", false, "Set to true if mlpipeline API server serves over TLS.") - metadataTLSEnabled = flag.Bool("metadata_tls_enabled", false, "Set to true if MLMD serves over TLS.") - caCertPath = flag.String("ca_cert_path", "", "The path to the CA certificate to trust on connections to the ML pipeline API server and metadata server.") -) - -// func RootDAG(pipelineName string, runID string, component *pipelinespec.ComponentSpec, task *pipelinespec.PipelineTaskSpec, mlmd *metadata.Client) (*Execution, error) { - -func main() { - flag.Parse() - - glog.Infof("Setting log level to: '%s'", *logLevel) - err := flag.Set("v", *logLevel) - if err != nil { - glog.Warningf("Failed to set log level: %s", err.Error()) - } - - err = drive() - if err != nil { - glog.Exitf("%v", err) - } -} - -// Use WARNING default logging level to facilitate troubleshooting. -func init() { - flag.Set("logtostderr", "true") - // Change the WARNING to INFO level for debugging. - flag.Set("stderrthreshold", "WARNING") -} - -func validate() error { - if *driverType == "" { - return fmt.Errorf("argument --%s must be specified", driverTypeArg) - } - if *httpProxy == unsetProxyArgValue { - return fmt.Errorf("argument --%s is required but can be an empty value", httpProxyArg) - } - if *httpsProxy == unsetProxyArgValue { - return fmt.Errorf("argument --%s is required but can be an empty value", httpsProxyArg) - } - if *noProxy == unsetProxyArgValue { - return fmt.Errorf("argument --%s is required but can be an empty value", noProxyArg) - } - // validation responsibility lives in driver itself, so we do not validate all other args - return nil -} - -func drive() (err error) { - defer func() { - if err != nil { - err = fmt.Errorf("KFP driver: %w", err) - } - }() - ctx := context.Background() - if err = validate(); err != nil { - return err - } - - // Support reading component spec from a file if value starts with @ - // This bypasses exec() argument size limits for large workflows - if strings.HasPrefix(*componentSpecJson, "@") { - filePath := (*componentSpecJson)[1:] // Remove the "@" prefix - data, err := os.ReadFile(filePath) - if err != nil { - return fmt.Errorf("failed to read component spec from file %s: %w", filePath, err) - } - *componentSpecJson = string(data) - glog.Infof("Read component spec from file: %s (%d bytes)", filePath, len(data)) - } - - proxy.InitializeConfig(*httpProxy, *httpsProxy, *noProxy) - glog.Infof("input ComponentSpec:%s\n", prettyPrint(*componentSpecJson)) - componentSpec := &pipelinespec.ComponentSpec{} - if err := util.UnmarshalString(*componentSpecJson, componentSpec); err != nil { - return fmt.Errorf("failed to unmarshal component spec, error: %w\ncomponentSpec: %v", err, prettyPrint(*componentSpecJson)) - } - var taskSpec *pipelinespec.PipelineTaskSpec - if *taskSpecJson != "" { - glog.Infof("input TaskSpec:%s\n", prettyPrint(*taskSpecJson)) - taskSpec = &pipelinespec.PipelineTaskSpec{} - if err := util.UnmarshalString(*taskSpecJson, taskSpec); err != nil { - return fmt.Errorf("failed to unmarshal task spec, error: %w\ntask: %v", err, taskSpecJson) - } - } - glog.Infof("input ContainerSpec:%s\n", prettyPrint(*containerSpecJson)) - containerSpec := &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{} - if err := util.UnmarshalString(*containerSpecJson, containerSpec); err != nil { - return fmt.Errorf("failed to unmarshal container spec, error: %w\ncontainerSpec: %v", err, containerSpecJson) - } - var runtimeConfig *pipelinespec.PipelineJob_RuntimeConfig - if *runtimeConfigJson != "" { - glog.Infof("input RuntimeConfig:%s\n", prettyPrint(*runtimeConfigJson)) - runtimeConfig = &pipelinespec.PipelineJob_RuntimeConfig{} - if err := util.UnmarshalString(*runtimeConfigJson, runtimeConfig); err != nil { - return fmt.Errorf("failed to unmarshal runtime config, error: %w\nruntimeConfig: %v", err, runtimeConfigJson) - } - } - k8sExecCfg, err := parseExecConfigJson(k8sExecConfigJson) - if err != nil { - return err - } - namespace, err := config.InPodNamespace() - if err != nil { - return err - } - var tlsCfg *tls.Config - if *metadataTLSEnabled { - tlsCfg, err = util.GetTLSConfig(*caCertPath) - if err != nil { - return err - } - } - client, err := newMlmdClient(*mlmdServerAddress, *mlmdServerPort, tlsCfg) - if err != nil { - return err - } - cacheClient, err := cacheutils.NewClient(*mlPipelineServerAddress, *mlPipelineServerPort, *cacheDisabledFlag, tlsCfg) - if err != nil { - return err - } - options := driver.Options{ - PipelineName: *pipelineName, - RunID: *runID, - RunName: *runName, - RunDisplayName: *runDisplayName, - Namespace: namespace, - Component: componentSpec, - Task: taskSpec, - DAGExecutionID: *dagExecutionID, - IterationIndex: *iterationIndex, - PipelineLogLevel: *logLevel, - PublishLogs: *publishLogs, - CacheDisabled: *cacheDisabledFlag, - DriverType: *driverType, - TaskName: *taskName, - MLPipelineServerAddress: *mlPipelineServerAddress, - MLPipelineServerPort: *mlPipelineServerPort, - MLMDServerAddress: *mlmdServerAddress, - MLMDServerPort: *mlmdServerPort, - MLPipelineTLSEnabled: *mlPipelineTLSEnabled, - MLMDTLSEnabled: *metadataTLSEnabled, - CaCertPath: *caCertPath, - } - var execution *driver.Execution - var driverErr error - switch *driverType { - case ROOT_DAG: - options.RuntimeConfig = runtimeConfig - execution, driverErr = driver.RootDAG(ctx, options, client) - case DAG: - execution, driverErr = driver.DAG(ctx, options, client) - case CONTAINER: - options.Container = containerSpec - options.KubernetesExecutorConfig = k8sExecCfg - execution, driverErr = driver.Container(ctx, options, client, cacheClient) - default: - err = fmt.Errorf("unknown driverType %s", *driverType) - } - if driverErr != nil { - if execution == nil { - return driverErr - } - defer func() { - // Override error with driver error, because driver error is more important. - // However, we continue running, because the following code prints debug info that - // may be helpful for figuring out why this failed. - err = driverErr - }() - } - - executionPaths := &ExecutionPaths{ - ExecutionID: *executionIDPath, - IterationCount: *iterationCountPath, - CachedDecision: *cachedDecisionPath, - Condition: *conditionPath, - PodSpecPatch: *podSpecPatchPath, - } - - return handleExecution(execution, *driverType, executionPaths) -} - -func parseExecConfigJson(k8sExecConfigJson *string) (*kubernetesplatform.KubernetesExecutorConfig, error) { - var k8sExecCfg *kubernetesplatform.KubernetesExecutorConfig - if *k8sExecConfigJson != "" { - glog.Infof("input kubernetesConfig:%s\n", prettyPrint(*k8sExecConfigJson)) - k8sExecCfg = &kubernetesplatform.KubernetesExecutorConfig{} - if err := util.UnmarshalString(*k8sExecConfigJson, k8sExecCfg); err != nil { - return nil, fmt.Errorf("failed to unmarshal Kubernetes config, error: %w\nKubernetesConfig: %v", err, k8sExecConfigJson) - } - } - return k8sExecCfg, nil -} - -func handleExecution(execution *driver.Execution, driverType string, executionPaths *ExecutionPaths) error { - if execution.ID != 0 { - glog.Infof("output execution.ID=%v", execution.ID) - if executionPaths.ExecutionID != "" { - if err := writeFile(executionPaths.ExecutionID, []byte(fmt.Sprint(execution.ID))); err != nil { - return fmt.Errorf("failed to write execution ID to file: %w", err) - } - } - } - if execution.IterationCount != nil { - if err := writeFile(executionPaths.IterationCount, []byte(fmt.Sprintf("%v", *execution.IterationCount))); err != nil { - return fmt.Errorf("failed to write iteration count to file: %w", err) - } - } else { - if driverType == ROOT_DAG { - if err := writeFile(executionPaths.IterationCount, []byte("0")); err != nil { - return fmt.Errorf("failed to write iteration count to file: %w", err) - } - } - } - if execution.Cached != nil { - if err := writeFile(executionPaths.CachedDecision, []byte(strconv.FormatBool(*execution.Cached))); err != nil { - return fmt.Errorf("failed to write cached decision to file: %w", err) - } - } - if execution.Condition != nil { - if err := writeFile(executionPaths.Condition, []byte(strconv.FormatBool(*execution.Condition))); err != nil { - return fmt.Errorf("failed to write condition to file: %w", err) - } - } else { - // nil is a valid value for Condition - if driverType == ROOT_DAG || driverType == CONTAINER { - if err := writeFile(executionPaths.Condition, []byte("nil")); err != nil { - return fmt.Errorf("failed to write condition to file: %w", err) - } - } - } - if execution.PodSpecPatch != "" { - glog.Infof("output podSpecPatch=\n%s\n", execution.PodSpecPatch) - if executionPaths.PodSpecPatch == "" { - return fmt.Errorf("--pod_spec_patch_path is required for container executor drivers") - } - if err := writeFile(executionPaths.PodSpecPatch, []byte(execution.PodSpecPatch)); err != nil { - return fmt.Errorf("failed to write pod spec patch to file: %w", err) - } - } - if execution.ExecutorInput != nil { - executorInputBytes, err := protojson.Marshal(execution.ExecutorInput) - if err != nil { - return fmt.Errorf("failed to marshal ExecutorInput to JSON: %w", err) - } - executorInputJSON := string(executorInputBytes) - glog.Infof("output ExecutorInput:%s\n", prettyPrint(executorInputJSON)) - } - return nil -} - -func prettyPrint(jsonStr string) string { - var prettyJSON bytes.Buffer - err := json.Indent(&prettyJSON, []byte(jsonStr), "", " ") - if err != nil { - return jsonStr - } - return prettyJSON.String() -} - -func writeFile(path string, data []byte) (err error) { - if path == "" { - return fmt.Errorf("path is not specified") - } - defer func() { - if err != nil { - err = fmt.Errorf("failed to write to %s: %w", path, err) - } - }() - if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { - return err - } - return os.WriteFile(path, data, 0o644) -} - -func newMlmdClient(mlmdServerAddress string, mlmdServerPort string, tlsCfg *tls.Config) (*metadata.Client, error) { - return metadata.NewClient(mlmdServerAddress, mlmdServerPort, tlsCfg) -} diff --git a/backend/src/v2/compiler/argocompiler/container.go b/backend/src/v2/compiler/argocompiler/container.go index 94e253703fa..6ed7a337eff 100644 --- a/backend/src/v2/compiler/argocompiler/container.go +++ b/backend/src/v2/compiler/argocompiler/container.go @@ -22,7 +22,6 @@ import ( "strings" "github.com/kubeflow/pipelines/backend/src/v2/config" - "github.com/kubeflow/pipelines/backend/src/v2/metadata" "google.golang.org/protobuf/encoding/protojson" "github.com/kubeflow/pipelines/backend/src/apiserver/config/proxy" @@ -178,31 +177,31 @@ func (c *workflowCompiler) addContainerDriverTemplate() (string, error) { } args := map[string]interface{}{ - "type": "CONTAINER", - "pipeline_name": c.spec.GetPipelineInfo().GetName(), - "run_id": runID(), - "run_name": runResourceName(), - "run_display_name": c.job.DisplayName, - "dag_execution_id": inputValue(paramParentDagID), - "component": inputValue(paramComponent), - "task": inputValue(paramTask), - "task_name": inputValue(paramTaskName), - "container": inputValue(paramContainer), - "iteration_index": inputValue(paramIterationIndex), - "cached_decision_path": outputPath(paramCachedDecision), - "pod_spec_patch_path": outputPath(paramPodSpecPatch), - "condition_path": outputPath(paramCondition), - "kubernetes_config": inputValue(paramKubernetesConfig), - "http_proxy": proxy.GetConfig().GetHttpProxy(), - "https_proxy": proxy.GetConfig().GetHttpsProxy(), - "no_proxy": proxy.GetConfig().GetNoProxy(), - "ml_pipeline_server_address", config.GetMLPipelineServerConfig().Address, - "ml_pipeline_server_port", config.GetMLPipelineServerConfig().Port, - "mlmd_server_address": metadata.DefaultConfig().Address, - "mlmd_server_port": metadata.DefaultConfig().Port, - "cache_disabled": c.cacheDisabled, - "ml_pipeline_tls_enabled": c.mlPipelineTLSEnabled, - "metadata_tls_enabled": common.GetMetadataTLSEnabled(), + "type": "CONTAINER", + "pipeline_name": c.spec.GetPipelineInfo().GetName(), + "run_id": runID(), + "run_name": runResourceName(), + "run_display_name": c.job.DisplayName, + "dag_execution_id": inputValue(paramParentDagID), + "component": inputValue(paramComponent), + "task": inputValue(paramTask), + "task_name": inputValue(paramTaskName), + "container": inputValue(paramContainer), + "iteration_index": inputValue(paramIterationIndex), + "cached_decision_path": outputPath(paramCachedDecision), + "pod_spec_patch_path": outputPath(paramPodSpecPatch), + "condition_path": outputPath(paramCondition), + "kubernetes_config": inputValue(paramKubernetesConfig), + "http_proxy": proxy.GetConfig().GetHttpProxy(), + "https_proxy": proxy.GetConfig().GetHttpsProxy(), + "no_proxy": proxy.GetConfig().GetNoProxy(), + "ml_pipeline_server_address": config.GetMLPipelineServerConfig().Address, + "ml_pipeline_server_port": config.GetMLPipelineServerConfig().Port, + "mlmd_server_address": config.GetMLPipelineServerConfig().Address, + "mlmd_server_port": config.GetMLPipelineServerConfig().Port, + "cache_disabled": c.cacheDisabled, + "ml_pipeline_tls_enabled": c.mlPipelineTLSEnabled, + "metadata_tls_enabled": common.GetMetadataTLSEnabled(), } setCABundle := false diff --git a/backend/src/v2/compiler/argocompiler/dag.go b/backend/src/v2/compiler/argocompiler/dag.go index db0c3e9f606..a44b089c33c 100644 --- a/backend/src/v2/compiler/argocompiler/dag.go +++ b/backend/src/v2/compiler/argocompiler/dag.go @@ -19,14 +19,12 @@ import ( "sort" "strings" - "github.com/kubeflow/pipelines/backend/src/apiserver/config/proxy" - "github.com/kubeflow/pipelines/backend/src/v2/config" - "github.com/kubeflow/pipelines/backend/src/v2/metadata" - wfapi "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" "github.com/kubeflow/pipelines/backend/src/apiserver/common" + "github.com/kubeflow/pipelines/backend/src/apiserver/config/proxy" "github.com/kubeflow/pipelines/backend/src/v2/compiler" + "github.com/kubeflow/pipelines/backend/src/v2/config" "k8s.io/apimachinery/pkg/util/intstr" ) @@ -562,30 +560,30 @@ func (c *workflowCompiler) addDAGDriverTemplate() (string, error) { } args := map[string]interface{}{ - "type": inputValue(paramDriverType), - "pipeline_name": c.spec.GetPipelineInfo().GetName(), - "run_id": runID(), - "run_name": runResourceName(), - "run_display_name": c.job.DisplayName, - "dag_execution_id": inputValue(paramParentDagID), - "component": inputValue(paramComponent), - "task": inputValue(paramTask), - "task_name": inputValue(paramTaskName), - "runtime_config": inputValue(paramRuntimeConfig), - "iteration_index": inputValue(paramIterationIndex), - "execution_id_path": outputPath(paramExecutionID), - "iteration_count_path": outputPath(paramIterationCount), - "condition_path": outputPath(paramCondition), - "http_proxy": proxy.GetConfig().GetHttpProxy(), - "https_proxy": proxy.GetConfig().GetHttpsProxy(), - "no_proxy": proxy.GetConfig().GetNoProxy(), - "ml_pipeline_server_address", config.GetMLPipelineServerConfig().Address, - "ml_pipeline_server_port", config.GetMLPipelineServerConfig().Port, - "mlmd_server_address": metadata.DefaultConfig().Address, - "mlmd_server_port": metadata.DefaultConfig().Port, - "cache_disabled": c.cacheDisabled, - "ml_pipeline_tls_enabled": c.mlPipelineTLSEnabled, - "metadata_tls_enabled": common.GetMetadataTLSEnabled(), + "type": inputValue(paramDriverType), + "pipeline_name": c.spec.GetPipelineInfo().GetName(), + "run_id": runID(), + "run_name": runResourceName(), + "run_display_name": c.job.DisplayName, + "dag_execution_id": inputValue(paramParentDagID), + "component": inputValue(paramComponent), + "task": inputValue(paramTask), + "task_name": inputValue(paramTaskName), + "runtime_config": inputValue(paramRuntimeConfig), + "iteration_index": inputValue(paramIterationIndex), + "execution_id_path": outputPath(paramExecutionID), + "iteration_count_path": outputPath(paramIterationCount), + "condition_path": outputPath(paramCondition), + "http_proxy": proxy.GetConfig().GetHttpProxy(), + "https_proxy": proxy.GetConfig().GetHttpsProxy(), + "no_proxy": proxy.GetConfig().GetNoProxy(), + "ml_pipeline_server_address": config.GetMLPipelineServerConfig().Address, + "ml_pipeline_server_port": config.GetMLPipelineServerConfig().Port, + "mlmd_server_address": config.GetMLPipelineServerConfig().Address, + "mlmd_server_port": config.GetMLPipelineServerConfig().Port, + "cache_disabled": c.cacheDisabled, + "ml_pipeline_tls_enabled": c.mlPipelineTLSEnabled, + "metadata_tls_enabled": common.GetMetadataTLSEnabled(), } setCABundle := false diff --git a/backend/test/compiler/argo_ginkgo_test.go b/backend/test/compiler/argo_ginkgo_test.go index 32b35194661..f6018b1d6c8 100644 --- a/backend/test/compiler/argo_ginkgo_test.go +++ b/backend/test/compiler/argo_ginkgo_test.go @@ -64,16 +64,18 @@ var _ = Describe("Verify Spec Compilation to Workflow >", Label(POSITIVE, Workfl envVars: map[string]string{"PIPELINE_RUN_AS_USER": "1001", "PIPELINE_LOG_LEVEL": "3"}, pipelineFilePaths: []string{filepath.Join(pipelineFilesRootDir, pipelineDirectory, "run_as_user_cache_enabled.yaml")}, }, - { - compilerOptions: argocompiler.Options{CacheDisabled: false}, - envVars: map[string]string{"CABUNDLE_CONFIGMAP_NAME": "test-configmap-name", "CABUNDLE_KEY_NAME": "test-configmap-key"}, - pipelineFilePaths: []string{filepath.Join(pipelineFilesRootDir, pipelineDirectory, "mounted_cabundle_configmap.yaml")}, - }, - { - compilerOptions: argocompiler.Options{CacheDisabled: false}, - envVars: map[string]string{"CABUNDLE_SECRET_NAME": "test-secret-name"}, - pipelineFilePaths: []string{filepath.Join(pipelineFilesRootDir, pipelineDirectory, "mounted_cabundle_secret.yaml")}, - }, + // WARN: Tests are temporarily ignored because volume mount management + // is not available at the plugin level. Plugin volumes can only be configured globally. + //{ + // compilerOptions: argocompiler.Options{CacheDisabled: false}, + // envVars: map[string]string{"CABUNDLE_CONFIGMAP_NAME": "test-configmap-name", "CABUNDLE_KEY_NAME": "test-configmap-key"}, + // pipelineFilePaths: []string{filepath.Join(pipelineFilesRootDir, pipelineDirectory, "mounted_cabundle_configmap.yaml")}, + //}, + //{ + // compilerOptions: argocompiler.Options{CacheDisabled: false}, + // envVars: map[string]string{"CABUNDLE_SECRET_NAME": "test-secret-name"}, + // pipelineFilePaths: []string{filepath.Join(pipelineFilesRootDir, pipelineDirectory, "mounted_cabundle_secret.yaml")}, + //}, } for _, param := range testParams { Context(fmt.Sprintf("Verify compiled workflow for a pipeline with compiler options cacheDisabled '%v' and env vars %v >", param.compilerOptions.CacheDisabled, param.envVars), Ordered, func() { diff --git a/backend/test/end2end/e2e_suite_test.go b/backend/test/end2end/e2e_suite_test.go index 0480694ffbe..621fb6de0e7 100644 --- a/backend/test/end2end/e2e_suite_test.go +++ b/backend/test/end2end/e2e_suite_test.go @@ -135,7 +135,6 @@ var _ = BeforeSuite(func() { }) var _ = BeforeEach(func() { - // Create Experiment so that we can use it to associate pipeline runs with experimentName := fmt.Sprintf("E2EExperiment-%s", strconv.FormatInt(time.Now().UnixNano(), 10)) experiment := testutil.CreateExperiment(experimentClient, experimentName, testutil.GetNamespace()) diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml index 98f9552b8b1..b5b05500b0d 100644 --- a/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml +++ b/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml @@ -132,7 +132,7 @@ spec: # JSON patch to apply to compiled workflow specifications - name: COMPILED_PIPELINE_SPEC_PATCH value: "{}" - image: ntny/kfp-api-server:beta-poc + image: ntny/kfp-api-server:beta-poc04 imagePullPolicy: IfNotPresent name: ml-pipeline-api-server ports: diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml index d116a40b47c..eaae5acc2d0 100644 --- a/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml +++ b/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml @@ -7,7 +7,7 @@ metadata: data: sidecar.automountServiceAccountToken: "true" sidecar.container: | - image: ntny/kfp-driver:beta-poc + image: ntny/kfp-driver:beta-poc04 name: driver-plugin ports: - containerPort: 8080 From db3ad727690e9cbcb1e501fdfbbaadcd8963f1e6 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Thu, 15 Jan 2026 14:37:58 +0300 Subject: [PATCH 08/51] Fixes for review comments, part 1 (easy ones) Signed-off-by: arpechenin --- backend/src/v2/compiler/argocompiler/container.go | 2 ++ backend/src/v2/compiler/argocompiler/dag.go | 2 ++ backend/src/v2/compiler/argocompiler/plugin.go | 2 ++ backend/test/compiler/argo_ginkgo_test.go | 12 ++++++------ backend/test/compiler/utils/workflow_utils.go | 1 + .../pipeline/ml-pipeline-apiserver-deployment.yaml | 2 +- .../base/pipeline/ml-pipeline-driver-plugin-cm.yaml | 2 ++ .../base/pipeline/pipeline-runner-role.yaml | 2 +- manifests/kustomize/env/dev/kustomization.yaml | 2 ++ .../application-controller-deployment.yaml | 2 +- .../base/workflow-controller-deployment-patch.yaml | 3 +++ .../argo/installs/namespace/kustomization.yaml | 3 +++ 12 files changed, 26 insertions(+), 9 deletions(-) diff --git a/backend/src/v2/compiler/argocompiler/container.go b/backend/src/v2/compiler/argocompiler/container.go index 6ed7a337eff..7b2743a876c 100644 --- a/backend/src/v2/compiler/argocompiler/container.go +++ b/backend/src/v2/compiler/argocompiler/container.go @@ -169,6 +169,8 @@ func (c *workflowCompiler) containerDriverTask(name string, inputs containerDriv return dagTask, outputs, nil } +// Create the Argo Workflow executor plugin template for the container driver. +// See https://argo-workflows.readthedocs.io/en/latest/executor_plugins/ func (c *workflowCompiler) addContainerDriverTemplate() (string, error) { name := "system-container-driver" _, ok := c.templates[name] diff --git a/backend/src/v2/compiler/argocompiler/dag.go b/backend/src/v2/compiler/argocompiler/dag.go index a44b089c33c..01d027e5ed4 100644 --- a/backend/src/v2/compiler/argocompiler/dag.go +++ b/backend/src/v2/compiler/argocompiler/dag.go @@ -552,6 +552,8 @@ func (c *workflowCompiler) dagDriverTask(name string, inputs dagDriverInputs) (* }, nil } +// Create the Argo Workflow executor plugin template for the dag driver. +// See https://argo-workflows.readthedocs.io/en/latest/executor_plugins/ func (c *workflowCompiler) addDAGDriverTemplate() (string, error) { name := "system-dag-driver" _, ok := c.templates[name] diff --git a/backend/src/v2/compiler/argocompiler/plugin.go b/backend/src/v2/compiler/argocompiler/plugin.go index a202c40270a..eb5f379a6dc 100644 --- a/backend/src/v2/compiler/argocompiler/plugin.go +++ b/backend/src/v2/compiler/argocompiler/plugin.go @@ -21,6 +21,8 @@ import ( wfapi "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" ) +// Create the Argo Workflow executor plugin template with parameters. +// // See https://argo-workflows.readthedocs.io/en/latest/executor_plugins/ func driverPlugin(params map[string]interface{}) (*wfapi.Plugin, error) { pluginConfig := map[string]interface{}{ "driver-plugin": map[string]interface{}{ diff --git a/backend/test/compiler/argo_ginkgo_test.go b/backend/test/compiler/argo_ginkgo_test.go index f6018b1d6c8..f66d29ac436 100644 --- a/backend/test/compiler/argo_ginkgo_test.go +++ b/backend/test/compiler/argo_ginkgo_test.go @@ -66,16 +66,16 @@ var _ = Describe("Verify Spec Compilation to Workflow >", Label(POSITIVE, Workfl }, // WARN: Tests are temporarily ignored because volume mount management // is not available at the plugin level. Plugin volumes can only be configured globally. - //{ - // compilerOptions: argocompiler.Options{CacheDisabled: false}, + // { + // compilerOptions: argocompiler.Options{CacheDisabled: false}, // envVars: map[string]string{"CABUNDLE_CONFIGMAP_NAME": "test-configmap-name", "CABUNDLE_KEY_NAME": "test-configmap-key"}, // pipelineFilePaths: []string{filepath.Join(pipelineFilesRootDir, pipelineDirectory, "mounted_cabundle_configmap.yaml")}, - //}, - //{ + // }, + // { // compilerOptions: argocompiler.Options{CacheDisabled: false}, // envVars: map[string]string{"CABUNDLE_SECRET_NAME": "test-secret-name"}, - // pipelineFilePaths: []string{filepath.Join(pipelineFilesRootDir, pipelineDirectory, "mounted_cabundle_secret.yaml")}, - //}, + // pipelineFilePaths: []string{filepath.Join(pipelineFilesRootDir, pipelineDirectory, "mounted_cabundle_secret.yaml")}, + // }, } for _, param := range testParams { Context(fmt.Sprintf("Verify compiled workflow for a pipeline with compiler options cacheDisabled '%v' and env vars %v >", param.compilerOptions.CacheDisabled, param.envVars), Ordered, func() { diff --git a/backend/test/compiler/utils/workflow_utils.go b/backend/test/compiler/utils/workflow_utils.go index a135cbc69cf..ebae3d2c31a 100644 --- a/backend/test/compiler/utils/workflow_utils.go +++ b/backend/test/compiler/utils/workflow_utils.go @@ -91,6 +91,7 @@ func CreateCompiledWorkflowFile(compiledWorflow *v1alpha1.Workflow, compiledWork return testutil.CreateFile(compiledWorkflowFilePath, [][]byte{fileContents}) } +// ConfigurePluginSettings - Add/Remove cache_disabled args in the driver-plugin func ConfigurePluginSettings(workflow *v1alpha1.Workflow, remove bool) *v1alpha1.Workflow { configuredWorkflow := workflow.DeepCopy() for i, template := range configuredWorkflow.Spec.Templates { diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml index b5b05500b0d..005dbe42885 100644 --- a/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml +++ b/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml @@ -132,7 +132,7 @@ spec: # JSON patch to apply to compiled workflow specifications - name: COMPILED_PIPELINE_SPEC_PATCH value: "{}" - image: ntny/kfp-api-server:beta-poc04 + image: ghcr.io/kubeflow/kfp-api-server:dummy imagePullPolicy: IfNotPresent name: ml-pipeline-api-server ports: diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml index eaae5acc2d0..e87bf138d0a 100644 --- a/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml +++ b/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml @@ -5,6 +5,8 @@ metadata: workflows.argoproj.io/configmap-type: ExecutorPlugin name: ml-pipeline-driver-agent data: + # The Argo Workflow executor plugin should be injected as a sidecar. + # Details: https://argo-workflows.readthedocs.io/en/latest/executor_plugins/#configuration sidecar.automountServiceAccountToken: "true" sidecar.container: | image: ntny/kfp-driver:beta-poc04 diff --git a/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml b/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml index 833885d0d0e..e76cba5b2e0 100644 --- a/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml +++ b/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml @@ -117,7 +117,7 @@ metadata: rules: - apiGroups: [""] resources: ["configmaps"] - verbs: ["get", "list", "watch"] # права на чтение + verbs: ["get", "list", "watch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding diff --git a/manifests/kustomize/env/dev/kustomization.yaml b/manifests/kustomize/env/dev/kustomization.yaml index b9dde28236f..a51b0c8728e 100644 --- a/manifests/kustomize/env/dev/kustomization.yaml +++ b/manifests/kustomize/env/dev/kustomization.yaml @@ -9,6 +9,8 @@ resources: - ../gcp/inverse-proxy images: +- name: ghcr.io/kubeflow/kfp-api-server + newTag: master - name: ghcr.io/kubeflow/kfp-driver newTag: master - name: ghcr.io/kubeflow/kfp-frontend diff --git a/manifests/kustomize/third-party/application/application-controller-deployment.yaml b/manifests/kustomize/third-party/application/application-controller-deployment.yaml index 2f4277da116..63497ef2c32 100644 --- a/manifests/kustomize/third-party/application/application-controller-deployment.yaml +++ b/manifests/kustomize/third-party/application/application-controller-deployment.yaml @@ -31,7 +31,7 @@ spec: resources: limits: cpu: 100m - memory: 128Mi + memory: 30Mi requests: cpu: 100m memory: 64Mi diff --git a/manifests/kustomize/third-party/argo/base/workflow-controller-deployment-patch.yaml b/manifests/kustomize/third-party/argo/base/workflow-controller-deployment-patch.yaml index d41d52359c6..d13da9e498f 100644 --- a/manifests/kustomize/third-party/argo/base/workflow-controller-deployment-patch.yaml +++ b/manifests/kustomize/third-party/argo/base/workflow-controller-deployment-patch.yaml @@ -18,8 +18,11 @@ spec: - --executor-image - quay.io/argoproj/argoexec:v3.7.3 env: + # https://argo-workflows.readthedocs.io/en/latest/environment-variables/ - name: ARGO_EXECUTOR_PLUGINS value: "true" + - name: DEFAULT_REQUEUE_TIME + value: "6s" securityContext: readOnlyRootFilesystem: true runAsNonRoot: true diff --git a/manifests/kustomize/third-party/argo/installs/namespace/kustomization.yaml b/manifests/kustomize/third-party/argo/installs/namespace/kustomization.yaml index 52df92bed5a..28d36b235a3 100644 --- a/manifests/kustomize/third-party/argo/installs/namespace/kustomization.yaml +++ b/manifests/kustomize/third-party/argo/installs/namespace/kustomization.yaml @@ -16,6 +16,9 @@ patches: kind: Deployment name: workflow-controller version: v1 +# Argo Workflow and the Agent communicate via the new WorkflowTaskSet CR, +# so RBAC permissions are required for this resource. +# Details: https://argo-workflows.readthedocs.io/en/latest/http-template/#argo-agent-rbac - path: workflow-controller-argo-role-patch.json target: group: rbac.authorization.k8s.io From f9d05373e1a5c12f28069fbaa78d7834ff2da59a Mon Sep 17 00:00:00 2001 From: arpechenin Date: Fri, 16 Jan 2026 01:09:14 +0300 Subject: [PATCH 09/51] make driver-plugin image kustimizable Signed-off-by: arpechenin --- .../manifests/base/driver-plugin-cm-path.yaml | 21 +++++++++++++++++++ .../default/kustomization.yaml | 4 ++++ .../artifact-proxy/kustomization.yaml | 4 ++++ .../cache-disabled/kustomization.yaml | 4 ++++ .../multiuser/default/kustomization.yaml | 4 ++++ .../cache-disabled-proxy/kustomization.yaml | 4 ++++ .../cache-disabled/kustomization.yaml | 4 ++++ .../standalone/default/kustomization.yaml | 4 ++++ .../standalone/proxy/kustomization.yaml | 4 ++++ .../standalone/tls-enabled/kustomization.yaml | 4 ++++ .../ml-pipeline-driver-plugin-cm.yaml | 8 +++---- .../env/dev/driver-plugin-cm-path.yaml | 20 ++++++++++++++++++ .../kustomize/env/dev/kustomization.yaml | 1 + 13 files changed, 81 insertions(+), 5 deletions(-) create mode 100644 .github/resources/manifests/base/driver-plugin-cm-path.yaml create mode 100644 manifests/kustomize/env/dev/driver-plugin-cm-path.yaml diff --git a/.github/resources/manifests/base/driver-plugin-cm-path.yaml b/.github/resources/manifests/base/driver-plugin-cm-path.yaml new file mode 100644 index 00000000000..54744ee94db --- /dev/null +++ b/.github/resources/manifests/base/driver-plugin-cm-path.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ml-pipeline-driver-agent +data: + sidecar.container: | + name: driver-plugin + image: kind-registry:5000/driver + ports: + - containerPort: 8080 + resources: + limits: + cpu: "1" + memory: 1Gi + requests: + cpu: 250m + memory: 512Mi + securityContext: + runAsNonRoot: true + runAsUser: 65534 + diff --git a/.github/resources/manifests/kubernetes-native/default/kustomization.yaml b/.github/resources/manifests/kubernetes-native/default/kustomization.yaml index 20a754898c8..5000ca3a89a 100644 --- a/.github/resources/manifests/kubernetes-native/default/kustomization.yaml +++ b/.github/resources/manifests/kubernetes-native/default/kustomization.yaml @@ -42,6 +42,10 @@ patches: target: kind: Deployment name: ml-pipeline + - path: ../../base/driver-plugin-cm-path.yaml + target: + kind: ConfigMap + name: ml-pipeline-driver-agent - path: ../../base/grpc-specs.yaml target: kind: Deployment diff --git a/.github/resources/manifests/multiuser/artifact-proxy/kustomization.yaml b/.github/resources/manifests/multiuser/artifact-proxy/kustomization.yaml index 9fcf4146950..53d3bfb2abf 100644 --- a/.github/resources/manifests/multiuser/artifact-proxy/kustomization.yaml +++ b/.github/resources/manifests/multiuser/artifact-proxy/kustomization.yaml @@ -42,6 +42,10 @@ patches: target: kind: Deployment name: ml-pipeline + - path: ../../base/driver-plugin-cm-path.yaml + target: + kind: ConfigMap + name: ml-pipeline-driver-agent - path: ../../base/grpc-specs.yaml target: kind: Deployment diff --git a/.github/resources/manifests/multiuser/cache-disabled/kustomization.yaml b/.github/resources/manifests/multiuser/cache-disabled/kustomization.yaml index 135a8567d9d..02331380687 100644 --- a/.github/resources/manifests/multiuser/cache-disabled/kustomization.yaml +++ b/.github/resources/manifests/multiuser/cache-disabled/kustomization.yaml @@ -42,6 +42,10 @@ patches: target: kind: Deployment name: ml-pipeline + - path: ../../base/driver-plugin-cm-path.yaml + target: + kind: ConfigMap + name: ml-pipeline-driver-agent - path: cache-env.yaml target: kind: Deployment diff --git a/.github/resources/manifests/multiuser/default/kustomization.yaml b/.github/resources/manifests/multiuser/default/kustomization.yaml index 024a62abb6a..fb0ac09f3f7 100644 --- a/.github/resources/manifests/multiuser/default/kustomization.yaml +++ b/.github/resources/manifests/multiuser/default/kustomization.yaml @@ -42,6 +42,10 @@ patches: target: kind: Deployment name: ml-pipeline + - path: ../../base/driver-plugin-cm-path.yaml + target: + kind: ConfigMap + name: ml-pipeline-driver-agent - path: ../../base/grpc-specs.yaml target: kind: Deployment diff --git a/.github/resources/manifests/standalone/cache-disabled-proxy/kustomization.yaml b/.github/resources/manifests/standalone/cache-disabled-proxy/kustomization.yaml index 3e41d8ee530..f8950444cf1 100644 --- a/.github/resources/manifests/standalone/cache-disabled-proxy/kustomization.yaml +++ b/.github/resources/manifests/standalone/cache-disabled-proxy/kustomization.yaml @@ -9,3 +9,7 @@ patches: target: kind: Deployment name: ml-pipeline + - path: ../../base/driver-plugin-cm-path.yaml + target: + kind: ConfigMap + name: ml-pipeline-driver-agent diff --git a/.github/resources/manifests/standalone/cache-disabled/kustomization.yaml b/.github/resources/manifests/standalone/cache-disabled/kustomization.yaml index 9f6c63ef1e4..cc67d29b189 100644 --- a/.github/resources/manifests/standalone/cache-disabled/kustomization.yaml +++ b/.github/resources/manifests/standalone/cache-disabled/kustomization.yaml @@ -42,6 +42,10 @@ patches: target: kind: Deployment name: ml-pipeline + - path: ../../base/driver-plugin-cm-path.yaml + target: + kind: ConfigMap + name: ml-pipeline-driver-agent - path: cache-env.yaml target: kind: Deployment diff --git a/.github/resources/manifests/standalone/default/kustomization.yaml b/.github/resources/manifests/standalone/default/kustomization.yaml index 8572e632121..318abe08741 100644 --- a/.github/resources/manifests/standalone/default/kustomization.yaml +++ b/.github/resources/manifests/standalone/default/kustomization.yaml @@ -42,6 +42,10 @@ patches: target: kind: Deployment name: ml-pipeline + - path: ../../base/driver-plugin-cm-path.yaml + target: + kind: ConfigMap + name: ml-pipeline-driver-agent - path: ../../base/grpc-specs.yaml target: kind: Deployment diff --git a/.github/resources/manifests/standalone/proxy/kustomization.yaml b/.github/resources/manifests/standalone/proxy/kustomization.yaml index 4d432795d1e..fac247137b3 100644 --- a/.github/resources/manifests/standalone/proxy/kustomization.yaml +++ b/.github/resources/manifests/standalone/proxy/kustomization.yaml @@ -46,6 +46,10 @@ patches: target: kind: Deployment name: ml-pipeline + - path: ../../base/driver-plugin-cm-path.yaml + target: + kind: ConfigMap + name: ml-pipeline-driver-agent - path: ../../base/grpc-specs.yaml target: kind: Deployment diff --git a/.github/resources/manifests/standalone/tls-enabled/kustomization.yaml b/.github/resources/manifests/standalone/tls-enabled/kustomization.yaml index f7dba7acc6e..8b5c80e2eff 100644 --- a/.github/resources/manifests/standalone/tls-enabled/kustomization.yaml +++ b/.github/resources/manifests/standalone/tls-enabled/kustomization.yaml @@ -46,6 +46,10 @@ patches: target: kind: Deployment name: ml-pipeline + - path: ../../base/driver-plugin-cm-path.yaml + target: + kind: ConfigMap + name: ml-pipeline-driver-agent - path: ../../base/grpc-specs.yaml target: kind: Deployment diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml index e87bf138d0a..d34e659d6b4 100644 --- a/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml +++ b/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml @@ -1,18 +1,16 @@ apiVersion: v1 kind: ConfigMap metadata: + name: ml-pipeline-driver-agent labels: workflows.argoproj.io/configmap-type: ExecutorPlugin - name: ml-pipeline-driver-agent data: - # The Argo Workflow executor plugin should be injected as a sidecar. - # Details: https://argo-workflows.readthedocs.io/en/latest/executor_plugins/#configuration sidecar.automountServiceAccountToken: "true" sidecar.container: | - image: ntny/kfp-driver:beta-poc04 name: driver-plugin + image: ghcr.io/kubeflow/kfp-driver:dummy ports: - - containerPort: 8080 + - containerPort: 8080 resources: limits: cpu: "1" diff --git a/manifests/kustomize/env/dev/driver-plugin-cm-path.yaml b/manifests/kustomize/env/dev/driver-plugin-cm-path.yaml new file mode 100644 index 00000000000..15951f419bc --- /dev/null +++ b/manifests/kustomize/env/dev/driver-plugin-cm-path.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ml-pipeline-driver-agent +data: + sidecar.container: | + name: driver-plugin + image: ghcr.io/kubeflow/kfp-driver:master + ports: + - containerPort: 8080 + resources: + limits: + cpu: "1" + memory: 1Gi + requests: + cpu: 250m + memory: 512Mi + securityContext: + runAsNonRoot: true + runAsUser: 65534 \ No newline at end of file diff --git a/manifests/kustomize/env/dev/kustomization.yaml b/manifests/kustomize/env/dev/kustomization.yaml index a51b0c8728e..50a263096b8 100644 --- a/manifests/kustomize/env/dev/kustomization.yaml +++ b/manifests/kustomize/env/dev/kustomization.yaml @@ -36,6 +36,7 @@ images: patches: - path: api-server-patch.yaml + - path: driver-plugin-cm-path.yaml # !!! If you want to customize the namespace, # please refer sample/cluster-scoped-resources to update the namespace for cluster-scoped-resources namespace: kubeflow From e5366400dcd15e67a527b06cda0750fd78e2fc7d Mon Sep 17 00:00:00 2001 From: arpechenin Date: Mon, 2 Feb 2026 00:25:19 +0300 Subject: [PATCH 10/51] Mount certificate to executor-plugin based drivers Signed-off-by: arpechenin --- backend/src/apiserver/common/const.go | 4 + backend/src/driver/main.go | 6 + .../src/v2/compiler/argocompiler/container.go | 8 +- backend/src/v2/compiler/argocompiler/dag.go | 11 +- backend/test/compiler/argo_ginkgo_test.go | 22 +- .../base-tls-certs/kfp-api-cert.yaml | 28 +++ test_data/compiled-workflows/add_numbers.yaml | 188 +++++---------- .../arguments_parameters.yaml | 188 +++++---------- .../compiled-workflows/artifact_cache.yaml | 188 +++++---------- .../compiled-workflows/artifact_crust.yaml | 188 +++++---------- .../compiled-workflows/artifacts_complex.yaml | 188 +++++---------- .../compiled-workflows/artifacts_simple.yaml | 188 +++++---------- .../collected_artifacts.yaml | 188 +++++---------- .../collected_parameters.yaml | 188 +++++---------- .../component_with_metadata_fields.yaml | 188 +++++---------- .../component_with_optional_inputs.yaml | 188 +++++---------- .../component_with_pip_index_urls.yaml | 188 +++++---------- .../component_with_pip_install.yaml | 188 +++++---------- .../component_with_pip_install_in_venv.yaml | 188 +++++---------- .../components_with_optional_artifacts.yaml | 188 +++++---------- .../compiled-workflows/concat_message.yaml | 188 +++++---------- .../conditional_producer_and_consumers.yaml | 188 +++++---------- .../container_component_with_no_inputs.yaml | 188 +++++---------- .../compiled-workflows/container_io.yaml | 188 +++++---------- .../container_no_input.yaml | 188 +++++---------- .../container_with_artifact_output.yaml | 188 +++++---------- .../container_with_concat_placeholder.yaml | 188 +++++---------- .../container_with_if_placeholder.yaml | 188 +++++---------- ...container_with_placeholder_in_fstring.yaml | 188 +++++---------- .../containerized_python_component.yaml | 188 +++++---------- .../create_pod_metadata_complex.yaml | 192 ++++++---------- .../cross_loop_after_topology.yaml | 188 +++++---------- test_data/compiled-workflows/dict_input.yaml | 188 +++++---------- .../compiled-workflows/embedded_artifact.yaml | 188 +++++---------- test_data/compiled-workflows/env-var.yaml | 188 +++++---------- test_data/compiled-workflows/fail_v2.yaml | 188 +++++---------- test_data/compiled-workflows/flip_coin.yaml | 188 +++++---------- test_data/compiled-workflows/hello_world.yaml | 188 +++++---------- test_data/compiled-workflows/identity.yaml | 188 +++++---------- .../if_elif_else_complex.yaml | 188 +++++---------- .../if_elif_else_with_oneof_parameters.yaml | 188 +++++---------- .../if_else_with_oneof_artifacts.yaml | 188 +++++---------- .../if_else_with_oneof_parameters.yaml | 188 +++++---------- .../compiled-workflows/input_artifact.yaml | 188 +++++---------- .../iris_pipeline_compiled.yaml | 188 +++++---------- ...lightweight_python_functions_pipeline.yaml | 188 +++++---------- ...tweight_python_functions_with_outputs.yaml | 188 +++++---------- .../log_streaming_compiled.yaml | 188 +++++---------- .../compiled-workflows/long-running.yaml | 188 +++++---------- .../loop_consume_upstream.yaml | 188 +++++---------- .../metrics_visualization_v2.yaml | 188 +++++---------- .../missing_kubernetes_optional_inputs.yaml | 188 +++++---------- .../compiled-workflows/mixed_parameters.yaml | 188 +++++---------- test_data/compiled-workflows/modelcar.yaml | 188 +++++---------- .../mounted_cabundle_configmap.yaml | 214 ++++++------------ .../mounted_cabundle_secret.yaml | 214 ++++++------------ .../multiple_artifacts_namedtuple.yaml | 188 +++++---------- .../multiple_parameters_namedtuple.yaml | 188 +++++---------- ...peline_opt_input_child_level_compiled.yaml | 188 +++++---------- ...sted_pipeline_opt_inputs_nil_compiled.yaml | 188 +++++---------- ...line_opt_inputs_parent_level_compiled.yaml | 188 +++++---------- .../compiled-workflows/nested_return.yaml | 188 +++++---------- .../nested_with_parameters.yaml | 188 +++++---------- .../notebook_component_mixed.yaml | 188 +++++---------- .../notebook_component_simple.yaml | 188 +++++---------- .../compiled-workflows/output_metrics.yaml | 188 +++++---------- .../parallel_for_after_dependency.yaml | 188 +++++---------- .../compiled-workflows/parameter_cache.yaml | 188 +++++---------- .../compiled-workflows/parameter_oneof.yaml | 188 +++++---------- .../parameters_complex.yaml | 188 +++++---------- .../compiled-workflows/parameters_simple.yaml | 188 +++++---------- .../pipeline_as_exit_task.yaml | 188 +++++---------- .../pipeline_in_pipeline.yaml | 188 +++++---------- .../pipeline_in_pipeline_complex.yaml | 188 +++++---------- ...pipeline_in_pipeline_loaded_from_yaml.yaml | 188 +++++---------- .../pipeline_producer_consumer.yaml | 188 +++++---------- .../pipeline_with_after.yaml | 188 +++++---------- ...ipeline_with_artifact_upload_download.yaml | 188 +++++---------- .../pipeline_with_concat_placeholder.yaml | 188 +++++---------- .../pipeline_with_condition.yaml | 188 +++++---------- ...namic_task_output_custom_training_job.yaml | 188 +++++---------- ...peline_with_dynamic_importer_metadata.yaml | 188 +++++---------- ...namic_task_output_custom_training_job.yaml | 188 +++++---------- .../compiled-workflows/pipeline_with_env.yaml | 188 +++++---------- .../pipeline_with_exit_handler.yaml | 188 +++++---------- .../pipeline_with_google_artifact_type.yaml | 188 +++++---------- .../pipeline_with_importer.yaml | 188 +++++---------- ...pipeline_with_importer_and_gcpc_types.yaml | 188 +++++---------- .../pipeline_with_importer_workspace.yaml | 188 +++++---------- .../pipeline_with_input_status_state.yaml | 188 +++++---------- .../pipeline_with_loops.yaml | 188 +++++---------- .../pipeline_with_loops_and_conditions.yaml | 188 +++++---------- .../pipeline_with_metadata_fields.yaml | 188 +++++---------- .../pipeline_with_metrics_outputs.yaml | 188 +++++---------- .../pipeline_with_multiple_exit_handlers.yaml | 188 +++++---------- .../pipeline_with_nested_conditions.yaml | 188 +++++---------- .../pipeline_with_nested_conditions_yaml.yaml | 188 +++++---------- .../pipeline_with_nested_loops.yaml | 188 +++++---------- .../pipeline_with_only_display_name.yaml | 188 +++++---------- .../pipeline_with_outputs.yaml | 188 +++++---------- ...pipeline_with_parallelfor_parallelism.yaml | 188 +++++---------- ...ipeline_with_params_containing_format.yaml | 188 +++++---------- .../pipeline_with_placeholders.yaml | 188 +++++---------- .../pipeline_with_pod_metadata.yaml | 196 ++++++---------- .../pipeline_with_retry.yaml | 188 +++++---------- .../pipeline_with_reused_component.yaml | 188 +++++---------- .../pipeline_with_secret_as_env.yaml | 188 +++++---------- .../pipeline_with_secret_as_volume.yaml | 188 +++++---------- ..._string_machine_fields_pipeline_input.yaml | 188 +++++---------- ...ith_string_machine_fields_task_output.yaml | 196 ++++++---------- .../pipeline_with_submit_request.yaml | 188 +++++---------- .../pipeline_with_task_final_status.yaml | 188 +++++---------- .../pipeline_with_task_final_status_yaml.yaml | 188 +++++---------- ...th_task_using_ignore_upstream_failure.yaml | 188 +++++---------- .../pipeline_with_utils.yaml | 188 +++++---------- .../pipeline_with_various_io_types.yaml | 188 +++++---------- .../pipeline_with_volume.yaml | 188 +++++---------- .../pipeline_with_volume_no_cache.yaml | 188 +++++---------- .../pipeline_with_workspace.yaml | 188 +++++---------- ..._with_if_placeholder_none_input_value.yaml | 188 +++++---------- test_data/compiled-workflows/preprocess.yaml | 188 +++++---------- .../producer_consumer_param_pipeline.yaml | 188 +++++---------- test_data/compiled-workflows/pvc_mount.yaml | 188 +++++---------- .../pythonic_artifact_with_single_return.yaml | 188 +++++---------- .../pythonic_artifacts_test_pipeline.yaml | 188 +++++---------- ...onic_artifacts_with_list_of_artifacts.yaml | 188 +++++---------- ...honic_artifacts_with_multiple_returns.yaml | 188 +++++---------- .../ray_integration_compiled.yaml | 188 +++++---------- .../run_as_user_cache_disabled.yaml | 194 ++++++---------- .../run_as_user_cache_enabled.yaml | 194 ++++++---------- .../compiled-workflows/sequential_v1.yaml | 188 +++++---------- .../compiled-workflows/sequential_v2.yaml | 188 +++++---------- .../compiled-workflows/take_nap_compiled.yaml | 188 +++++---------- .../take_nap_pipeline_root_compiled.yaml | 188 +++++---------- .../compiled-workflows/two_step_pipeline.yaml | 188 +++++---------- .../two_step_pipeline_containerized.yaml | 188 +++++---------- .../upload_download_compiled.yaml | 188 +++++---------- .../xgboost_sample_pipeline.yaml | 188 +++++---------- 138 files changed, 8121 insertions(+), 16858 deletions(-) diff --git a/backend/src/apiserver/common/const.go b/backend/src/apiserver/common/const.go index 584c8e63a01..e317c7fd233 100644 --- a/backend/src/apiserver/common/const.go +++ b/backend/src/apiserver/common/const.go @@ -69,6 +69,10 @@ const ( const ( CustomCaCertPath = "/kfp/certs/ca.crt" + // DriverCaCertPath is the path to the client CA certificate. + // The Driver, based on the Argo Workflow Executor Plugin, expects the CA certificate key + // to be mounted from the Secret at this path: /etc/ssl/certs/ca-certificates/ca.crt. + DriverCaCertPath = "/etc/ssl/certs/ca-certificates/ca.crt" CABundleDir = "/kfp/certs" ) diff --git a/backend/src/driver/main.go b/backend/src/driver/main.go index c2bd03daddc..7c1ab369818 100644 --- a/backend/src/driver/main.go +++ b/backend/src/driver/main.go @@ -51,6 +51,12 @@ var ( mlmdServerPort = flag.String("mlmd_server_port", "8080", "MLMD server port") serverPort = flag.String("server_port", ":8080", "Server port") + + // CACertPath Path to the CA certificate + // Default "/etc/ssl/certs/ca-certificates/" is the path where the Argo executor sidecar expects the CA certificate. + // If a Secret named `argo-workflows-agent-ca-certificates` exists in the cluster, + // Argo will automatically mount it here in the sidecar container. + CACertPath = flag.String("ca_cert_path", "/etc/ssl/certs/ca-certificates/", "Path to the CA certificate") ) func main() { diff --git a/backend/src/v2/compiler/argocompiler/container.go b/backend/src/v2/compiler/argocompiler/container.go index 7b2743a876c..bb48698f971 100644 --- a/backend/src/v2/compiler/argocompiler/container.go +++ b/backend/src/v2/compiler/argocompiler/container.go @@ -206,11 +206,9 @@ func (c *workflowCompiler) addContainerDriverTemplate() (string, error) { "metadata_tls_enabled": common.GetMetadataTLSEnabled(), } - setCABundle := false // If CABUNDLE_SECRET_NAME or CABUNDLE_CONFIGMAP_NAME is set, add ca_cert_path arg to container driver. if common.GetCaBundleSecretName() != "" || common.GetCaBundleConfigMapName() != "" { - args["ca_cert_path"] = common.CustomCaCertPath - setCABundle = true + args["ca_cert_path"] = common.DriverCaCertPath } if value, ok := os.LookupEnv(PipelineLogLevelEnvVar); ok { @@ -248,10 +246,6 @@ func (c *workflowCompiler) addContainerDriverTemplate() (string, error) { Plugin: containerDriverPlugin, } applySecurityContextToTemplate(template) - // If TLS is enabled (apiserver or metadata), add the custom CA bundle to the container driver template. - if setCABundle { - ConfigureCustomCABundle(template) - } c.templates[name] = template c.wf.Spec.Templates = append(c.wf.Spec.Templates, *template) return name, err diff --git a/backend/src/v2/compiler/argocompiler/dag.go b/backend/src/v2/compiler/argocompiler/dag.go index 01d027e5ed4..66104266fb4 100644 --- a/backend/src/v2/compiler/argocompiler/dag.go +++ b/backend/src/v2/compiler/argocompiler/dag.go @@ -587,12 +587,9 @@ func (c *workflowCompiler) addDAGDriverTemplate() (string, error) { "ml_pipeline_tls_enabled": c.mlPipelineTLSEnabled, "metadata_tls_enabled": common.GetMetadataTLSEnabled(), } - - setCABundle := false - // If CABUNDLE_SECRET_NAME or CABUNDLE_CONFIGMAP_NAME is set, add ca_cert_path arg to DAG driver. + // If CABUNDLE_SECRET_NAME add ca_cert_path arg to DAG driver. if common.GetCaBundleSecretName() != "" || common.GetCaBundleConfigMapName() != "" { - args["ca_cert_path"] = common.CustomCaCertPath - setCABundle = true + args["ca_cert_path"] = common.DriverCaCertPath } if value, ok := os.LookupEnv(PipelineLogLevelEnvVar); ok { @@ -630,10 +627,6 @@ func (c *workflowCompiler) addDAGDriverTemplate() (string, error) { Plugin: dagPlugin, } applySecurityContextToTemplate(template) - // If TLS is enabled (apiserver or metadata), add the custom CA bundle to the DAG driver template. - if setCABundle { - ConfigureCustomCABundle(template) - } c.templates[name] = template c.wf.Spec.Templates = append(c.wf.Spec.Templates, *template) return name, nil diff --git a/backend/test/compiler/argo_ginkgo_test.go b/backend/test/compiler/argo_ginkgo_test.go index f66d29ac436..32b35194661 100644 --- a/backend/test/compiler/argo_ginkgo_test.go +++ b/backend/test/compiler/argo_ginkgo_test.go @@ -64,18 +64,16 @@ var _ = Describe("Verify Spec Compilation to Workflow >", Label(POSITIVE, Workfl envVars: map[string]string{"PIPELINE_RUN_AS_USER": "1001", "PIPELINE_LOG_LEVEL": "3"}, pipelineFilePaths: []string{filepath.Join(pipelineFilesRootDir, pipelineDirectory, "run_as_user_cache_enabled.yaml")}, }, - // WARN: Tests are temporarily ignored because volume mount management - // is not available at the plugin level. Plugin volumes can only be configured globally. - // { - // compilerOptions: argocompiler.Options{CacheDisabled: false}, - // envVars: map[string]string{"CABUNDLE_CONFIGMAP_NAME": "test-configmap-name", "CABUNDLE_KEY_NAME": "test-configmap-key"}, - // pipelineFilePaths: []string{filepath.Join(pipelineFilesRootDir, pipelineDirectory, "mounted_cabundle_configmap.yaml")}, - // }, - // { - // compilerOptions: argocompiler.Options{CacheDisabled: false}, - // envVars: map[string]string{"CABUNDLE_SECRET_NAME": "test-secret-name"}, - // pipelineFilePaths: []string{filepath.Join(pipelineFilesRootDir, pipelineDirectory, "mounted_cabundle_secret.yaml")}, - // }, + { + compilerOptions: argocompiler.Options{CacheDisabled: false}, + envVars: map[string]string{"CABUNDLE_CONFIGMAP_NAME": "test-configmap-name", "CABUNDLE_KEY_NAME": "test-configmap-key"}, + pipelineFilePaths: []string{filepath.Join(pipelineFilesRootDir, pipelineDirectory, "mounted_cabundle_configmap.yaml")}, + }, + { + compilerOptions: argocompiler.Options{CacheDisabled: false}, + envVars: map[string]string{"CABUNDLE_SECRET_NAME": "test-secret-name"}, + pipelineFilePaths: []string{filepath.Join(pipelineFilesRootDir, pipelineDirectory, "mounted_cabundle_secret.yaml")}, + }, } for _, param := range testParams { Context(fmt.Sprintf("Verify compiled workflow for a pipeline with compiler options cacheDisabled '%v' and env vars %v >", param.compilerOptions.CacheDisabled, param.envVars), Ordered, func() { diff --git a/manifests/kustomize/env/cert-manager/base-tls-certs/kfp-api-cert.yaml b/manifests/kustomize/env/cert-manager/base-tls-certs/kfp-api-cert.yaml index 8dcdf1c623c..f61d447f780 100644 --- a/manifests/kustomize/env/cert-manager/base-tls-certs/kfp-api-cert.yaml +++ b/manifests/kustomize/env/cert-manager/base-tls-certs/kfp-api-cert.yaml @@ -24,3 +24,31 @@ spec: kind: Issuer name: kfp-api-tls-selfsigned-issuer secretName: kfp-api-tls-cert +--- +# Necessary for mounting TLS CA certificate (created by cert-manager) to driver executor plugin +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: argo-workflows-agent-ca-certificates +spec: + commonName: argo-workflows-agent-ca-certificates + isCA: true + dnsNames: + - ml-pipeline + - ml-pipeline.kubeflow + - ml-pipeline.$(kfp-namespace).svc.cluster.local + - ml-pipeline-scheduledworkflow + - metadata-envoy + - metadata-envoy-service + - metadata-grpc-service + - metadata-grpc-service.kubeflow + - metadata-grpc-service.$(kfp-namespace).svc.cluster.local + # localhost included here because cert is used in KFP pod-to-pod TLS-enabled testing against localhost base URL + - localhost + ipAddresses: + # Necessary for running TLS-enabled cluster locally. + - 127.0.0.1 + issuerRef: + kind: Issuer + name: kfp-api-tls-selfsigned-issuer + secretName: argo-workflows-agent-ca-certificates \ No newline at end of file diff --git a/test_data/compiled-workflows/add_numbers.yaml b/test_data/compiled-workflows/add_numbers.yaml index 62aa216be99..fc0b147d446 100644 --- a/test_data/compiled-workflows/add_numbers.yaml +++ b/test_data/compiled-workflows/add_numbers.yaml @@ -29,73 +29,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - add-numbers - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -113,16 +47,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: add-numbers + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -185,7 +147,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -245,62 +207,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - add-numbers - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -330,6 +237,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: add-numbers + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/arguments_parameters.yaml b/test_data/compiled-workflows/arguments_parameters.yaml index b2a7f8cef12..749efba9187 100644 --- a/test_data/compiled-workflows/arguments_parameters.yaml +++ b/test_data/compiled-workflows/arguments_parameters.yaml @@ -20,73 +20,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -104,16 +38,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: echo + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -176,7 +138,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -236,62 +198,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -321,6 +228,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: echo + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/artifact_cache.yaml b/test_data/compiled-workflows/artifact_cache.yaml index 25957d23828..d439fc6c38d 100644 --- a/test_data/compiled-workflows/artifact_cache.yaml +++ b/test_data/compiled-workflows/artifact_cache.yaml @@ -48,73 +48,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - artifact-cache-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -132,16 +66,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: artifact-cache-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -204,7 +166,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -264,62 +226,7 @@ spec: metadata: {} name: comp-core outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - artifact-cache-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -349,6 +256,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: artifact-cache-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/artifact_crust.yaml b/test_data/compiled-workflows/artifact_crust.yaml index 2f29f578ed5..a0c5cc7c794 100644 --- a/test_data/compiled-workflows/artifact_crust.yaml +++ b/test_data/compiled-workflows/artifact_crust.yaml @@ -48,73 +48,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - artifact-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -132,16 +66,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: artifact-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -204,7 +166,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -264,62 +226,7 @@ spec: metadata: {} name: comp-core outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - artifact-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -349,6 +256,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: artifact-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/artifacts_complex.yaml b/test_data/compiled-workflows/artifacts_complex.yaml index e15983b569c..e75f349c59d 100644 --- a/test_data/compiled-workflows/artifacts_complex.yaml +++ b/test_data/compiled-workflows/artifacts_complex.yaml @@ -74,73 +74,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -158,16 +92,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: math-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -230,7 +192,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -371,62 +333,7 @@ spec: metadata: {} name: comp-condition-5 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -456,6 +363,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: math-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/artifacts_simple.yaml b/test_data/compiled-workflows/artifacts_simple.yaml index 14812f852c9..6684e132194 100644 --- a/test_data/compiled-workflows/artifacts_simple.yaml +++ b/test_data/compiled-workflows/artifacts_simple.yaml @@ -59,73 +59,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -143,16 +77,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: math-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -215,7 +177,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -275,62 +237,7 @@ spec: metadata: {} name: comp-for-loop-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -360,6 +267,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: math-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/collected_artifacts.yaml b/test_data/compiled-workflows/collected_artifacts.yaml index 451f34e2d6f..b876e2fe5b5 100644 --- a/test_data/compiled-workflows/collected_artifacts.yaml +++ b/test_data/compiled-workflows/collected_artifacts.yaml @@ -140,73 +140,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - collected-artifact-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -224,16 +158,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: collected-artifact-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -296,7 +258,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -356,62 +318,7 @@ spec: metadata: {} name: comp-single-node-dag outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - collected-artifact-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -441,6 +348,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: collected-artifact-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/collected_parameters.yaml b/test_data/compiled-workflows/collected_parameters.yaml index 73bc3c25aa6..28f87d480d6 100644 --- a/test_data/compiled-workflows/collected_parameters.yaml +++ b/test_data/compiled-workflows/collected_parameters.yaml @@ -75,73 +75,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - collected-param-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -159,16 +93,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: collected-param-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -231,7 +193,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -316,62 +278,7 @@ spec: metadata: {} name: comp-for-loop-1 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - collected-param-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -401,6 +308,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: collected-param-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/component_with_metadata_fields.yaml b/test_data/compiled-workflows/component_with_metadata_fields.yaml index e968493445c..2f8ed160cda 100644 --- a/test_data/compiled-workflows/component_with_metadata_fields.yaml +++ b/test_data/compiled-workflows/component_with_metadata_fields.yaml @@ -44,73 +44,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - dataset-joiner - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -128,16 +62,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: dataset-joiner + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -200,7 +162,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -260,62 +222,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - dataset-joiner - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -345,6 +252,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: dataset-joiner + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/component_with_optional_inputs.yaml b/test_data/compiled-workflows/component_with_optional_inputs.yaml index 49cb8425645..a9094726b48 100644 --- a/test_data/compiled-workflows/component_with_optional_inputs.yaml +++ b/test_data/compiled-workflows/component_with_optional_inputs.yaml @@ -33,73 +33,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - v2-component-optional-input - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -117,16 +51,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: v2-component-optional-input + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -189,7 +151,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -249,62 +211,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - v2-component-optional-input - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -334,6 +241,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: v2-component-optional-input + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/component_with_pip_index_urls.yaml b/test_data/compiled-workflows/component_with_pip_index_urls.yaml index c9e93836d19..bcd176cb5b0 100644 --- a/test_data/compiled-workflows/component_with_pip_index_urls.yaml +++ b/test_data/compiled-workflows/component_with_pip_index_urls.yaml @@ -31,73 +31,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - v2-component-pip-index-urls - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -115,16 +49,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: v2-component-pip-index-urls + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -187,7 +149,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -247,62 +209,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - v2-component-pip-index-urls - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -332,6 +239,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: v2-component-pip-index-urls + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/component_with_pip_install.yaml b/test_data/compiled-workflows/component_with_pip_install.yaml index 39f22d9cce1..4432b4be9f3 100644 --- a/test_data/compiled-workflows/component_with_pip_install.yaml +++ b/test_data/compiled-workflows/component_with_pip_install.yaml @@ -31,73 +31,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - component-with-pip-install - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -115,16 +49,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: component-with-pip-install + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -187,7 +149,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -247,62 +209,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - component-with-pip-install - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -332,6 +239,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: component-with-pip-install + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/component_with_pip_install_in_venv.yaml b/test_data/compiled-workflows/component_with_pip_install_in_venv.yaml index 7e23a511594..41c9a3d5b2c 100644 --- a/test_data/compiled-workflows/component_with_pip_install_in_venv.yaml +++ b/test_data/compiled-workflows/component_with_pip_install_in_venv.yaml @@ -32,73 +32,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - component-with-pip-install - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -116,16 +50,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: component-with-pip-install + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -188,7 +150,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -248,62 +210,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - component-with-pip-install - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -333,6 +240,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: component-with-pip-install + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/components_with_optional_artifacts.yaml b/test_data/compiled-workflows/components_with_optional_artifacts.yaml index af51d27f5c2..72124af42a3 100644 --- a/test_data/compiled-workflows/components_with_optional_artifacts.yaml +++ b/test_data/compiled-workflows/components_with_optional_artifacts.yaml @@ -45,73 +45,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - optional-artifact-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -129,16 +63,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: optional-artifact-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -201,7 +163,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -350,62 +312,7 @@ spec: metadata: {} name: system-importer outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - optional-artifact-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -435,6 +342,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: optional-artifact-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/concat_message.yaml b/test_data/compiled-workflows/concat_message.yaml index 442393ad338..8d2585a6b65 100644 --- a/test_data/compiled-workflows/concat_message.yaml +++ b/test_data/compiled-workflows/concat_message.yaml @@ -30,73 +30,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - concat-message - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -114,16 +48,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: concat-message + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -186,7 +148,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -246,62 +208,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - concat-message - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -331,6 +238,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: concat-message + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/conditional_producer_and_consumers.yaml b/test_data/compiled-workflows/conditional_producer_and_consumers.yaml index aea031644a3..1a6e2b4b2ef 100644 --- a/test_data/compiled-workflows/conditional_producer_and_consumers.yaml +++ b/test_data/compiled-workflows/conditional_producer_and_consumers.yaml @@ -51,73 +51,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -135,16 +69,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: math-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -207,7 +169,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -299,62 +261,7 @@ spec: metadata: {} name: comp-condition-3 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -384,6 +291,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: math-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/container_component_with_no_inputs.yaml b/test_data/compiled-workflows/container_component_with_no_inputs.yaml index f3097105158..ebc35475dc5 100644 --- a/test_data/compiled-workflows/container_component_with_no_inputs.yaml +++ b/test_data/compiled-workflows/container_component_with_no_inputs.yaml @@ -20,73 +20,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - v2-container-component-no-input - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -104,16 +38,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: v2-container-component-no-input + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -176,7 +138,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -236,62 +198,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - v2-container-component-no-input - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -321,6 +228,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: v2-container-component-no-input + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/container_io.yaml b/test_data/compiled-workflows/container_io.yaml index dfde886d3ac..e9f9ced178b 100644 --- a/test_data/compiled-workflows/container_io.yaml +++ b/test_data/compiled-workflows/container_io.yaml @@ -20,73 +20,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - container-io - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -104,16 +38,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: container-io + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -176,7 +138,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -236,62 +198,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - container-io - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -321,6 +228,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: container-io + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/container_no_input.yaml b/test_data/compiled-workflows/container_no_input.yaml index 80f7149d9dd..5bbb4e40e48 100644 --- a/test_data/compiled-workflows/container_no_input.yaml +++ b/test_data/compiled-workflows/container_no_input.yaml @@ -20,73 +20,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - container-no-input - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -104,16 +38,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: container-no-input + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -176,7 +138,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -236,62 +198,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - container-no-input - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -321,6 +228,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: container-no-input + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/container_with_artifact_output.yaml b/test_data/compiled-workflows/container_with_artifact_output.yaml index 41c5a8379ae..e948722513f 100644 --- a/test_data/compiled-workflows/container_with_artifact_output.yaml +++ b/test_data/compiled-workflows/container_with_artifact_output.yaml @@ -20,73 +20,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - container-with-artifact-output - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -104,16 +38,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: container-with-artifact-output + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -176,7 +138,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -236,62 +198,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - container-with-artifact-output - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -321,6 +228,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: container-with-artifact-output + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/container_with_concat_placeholder.yaml b/test_data/compiled-workflows/container_with_concat_placeholder.yaml index d32ab8a54e1..cb91b0f16cc 100644 --- a/test_data/compiled-workflows/container_with_concat_placeholder.yaml +++ b/test_data/compiled-workflows/container_with_concat_placeholder.yaml @@ -21,73 +21,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - container-with-concat-placeholder - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -105,16 +39,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: container-with-concat-placeholder + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -177,7 +139,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -237,62 +199,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - container-with-concat-placeholder - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -322,6 +229,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: container-with-concat-placeholder + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/container_with_if_placeholder.yaml b/test_data/compiled-workflows/container_with_if_placeholder.yaml index e902c6663f6..0c48844c104 100644 --- a/test_data/compiled-workflows/container_with_if_placeholder.yaml +++ b/test_data/compiled-workflows/container_with_if_placeholder.yaml @@ -23,73 +23,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - container-with-if-placeholder - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -107,16 +41,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: container-with-if-placeholder + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -179,7 +141,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -239,62 +201,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - container-with-if-placeholder - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -324,6 +231,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: container-with-if-placeholder + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/container_with_placeholder_in_fstring.yaml b/test_data/compiled-workflows/container_with_placeholder_in_fstring.yaml index 80b7b8cd82c..4ff8d8f59be 100644 --- a/test_data/compiled-workflows/container_with_placeholder_in_fstring.yaml +++ b/test_data/compiled-workflows/container_with_placeholder_in_fstring.yaml @@ -20,73 +20,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - container-with-placeholder-in-fstring - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -104,16 +38,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: container-with-placeholder-in-fstring + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -176,7 +138,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -236,62 +198,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - container-with-placeholder-in-fstring - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -321,6 +228,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: container-with-placeholder-in-fstring + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/containerized_python_component.yaml b/test_data/compiled-workflows/containerized_python_component.yaml index 873cbec6fd7..7ca099e2273 100644 --- a/test_data/compiled-workflows/containerized_python_component.yaml +++ b/test_data/compiled-workflows/containerized_python_component.yaml @@ -20,73 +20,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - concat-message - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -104,16 +38,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: concat-message + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -176,7 +138,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -236,62 +198,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - concat-message - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -321,6 +228,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: concat-message + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/create_pod_metadata_complex.yaml b/test_data/compiled-workflows/create_pod_metadata_complex.yaml index 6880b31dc24..c6f0e2274ef 100644 --- a/test_data/compiled-workflows/create_pod_metadata_complex.yaml +++ b/test_data/compiled-workflows/create_pod_metadata_complex.yaml @@ -73,73 +73,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-pod-metadata - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -157,16 +91,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-pod-metadata + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -229,7 +191,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -337,7 +299,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -450,7 +412,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -589,62 +551,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-pod-metadata - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -674,6 +581,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-pod-metadata + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/cross_loop_after_topology.yaml b/test_data/compiled-workflows/cross_loop_after_topology.yaml index ca031bb6cf4..1b2f9a16df2 100644 --- a/test_data/compiled-workflows/cross_loop_after_topology.yaml +++ b/test_data/compiled-workflows/cross_loop_after_topology.yaml @@ -50,73 +50,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - my-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -134,16 +68,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: my-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -206,7 +168,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -298,62 +260,7 @@ spec: metadata: {} name: comp-for-loop-14 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - my-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -383,6 +290,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: my-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/dict_input.yaml b/test_data/compiled-workflows/dict_input.yaml index 252192d0502..99376dec431 100644 --- a/test_data/compiled-workflows/dict_input.yaml +++ b/test_data/compiled-workflows/dict_input.yaml @@ -29,73 +29,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - dict-input - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -113,16 +47,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: dict-input + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -185,7 +147,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -245,62 +207,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - dict-input - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -330,6 +237,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: dict-input + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/embedded_artifact.yaml b/test_data/compiled-workflows/embedded_artifact.yaml index a8417bfab8f..5cd04e9bf68 100644 --- a/test_data/compiled-workflows/embedded_artifact.yaml +++ b/test_data/compiled-workflows/embedded_artifact.yaml @@ -70,73 +70,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - nb-simple - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -154,16 +88,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: nb-simple + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -226,7 +188,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -310,62 +272,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - nb-simple - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -395,6 +302,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: nb-simple + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/env-var.yaml b/test_data/compiled-workflows/env-var.yaml index e74b07771ee..ad16e797f76 100644 --- a/test_data/compiled-workflows/env-var.yaml +++ b/test_data/compiled-workflows/env-var.yaml @@ -31,73 +31,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - test-env-exists - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -115,16 +49,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: test-env-exists + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -187,7 +149,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -247,62 +209,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - test-env-exists - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -332,6 +239,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: test-env-exists + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/fail_v2.yaml b/test_data/compiled-workflows/fail_v2.yaml index 668b99db141..278898c0fb0 100644 --- a/test_data/compiled-workflows/fail_v2.yaml +++ b/test_data/compiled-workflows/fail_v2.yaml @@ -29,73 +29,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - fail-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -113,16 +47,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: fail-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -185,7 +147,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -245,62 +207,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - fail-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -330,6 +237,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: fail-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/flip_coin.yaml b/test_data/compiled-workflows/flip_coin.yaml index ebc4c9edea7..59c2ea3964a 100644 --- a/test_data/compiled-workflows/flip_coin.yaml +++ b/test_data/compiled-workflows/flip_coin.yaml @@ -86,73 +86,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - conditional-execution-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -170,16 +104,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: conditional-execution-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -242,7 +204,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -338,62 +300,7 @@ spec: metadata: {} name: comp-condition-3 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - conditional-execution-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -423,6 +330,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: conditional-execution-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/hello_world.yaml b/test_data/compiled-workflows/hello_world.yaml index e765d6b8fba..696efcb8d40 100644 --- a/test_data/compiled-workflows/hello_world.yaml +++ b/test_data/compiled-workflows/hello_world.yaml @@ -20,73 +20,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -104,16 +38,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: echo + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -176,7 +138,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -236,62 +198,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -321,6 +228,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: echo + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/identity.yaml b/test_data/compiled-workflows/identity.yaml index e8c911a4208..93e8fe36409 100644 --- a/test_data/compiled-workflows/identity.yaml +++ b/test_data/compiled-workflows/identity.yaml @@ -29,73 +29,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - identity - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -113,16 +47,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: identity + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -185,7 +147,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -245,62 +207,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - identity - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -330,6 +237,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: identity + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/if_elif_else_complex.yaml b/test_data/compiled-workflows/if_elif_else_complex.yaml index 9c73eabcb4d..05b4a3301de 100644 --- a/test_data/compiled-workflows/if_elif_else_complex.yaml +++ b/test_data/compiled-workflows/if_elif_else_complex.yaml @@ -120,73 +120,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - lucky-number-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -204,16 +138,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: lucky-number-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -276,7 +238,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -337,62 +299,7 @@ spec: metadata: {} name: comp-condition-3 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - lucky-number-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -422,6 +329,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: lucky-number-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/if_elif_else_with_oneof_parameters.yaml b/test_data/compiled-workflows/if_elif_else_with_oneof_parameters.yaml index 43c5c7ab509..cc5d02e1fe3 100644 --- a/test_data/compiled-workflows/if_elif_else_with_oneof_parameters.yaml +++ b/test_data/compiled-workflows/if_elif_else_with_oneof_parameters.yaml @@ -75,73 +75,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - outer-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -159,16 +93,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: outer-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -231,7 +193,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -357,62 +319,7 @@ spec: metadata: {} name: comp-condition-4 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - outer-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -442,6 +349,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: outer-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/if_else_with_oneof_artifacts.yaml b/test_data/compiled-workflows/if_else_with_oneof_artifacts.yaml index 385e39b07b6..1116b8d2a6e 100644 --- a/test_data/compiled-workflows/if_else_with_oneof_artifacts.yaml +++ b/test_data/compiled-workflows/if_else_with_oneof_artifacts.yaml @@ -67,73 +67,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - outer-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -151,16 +85,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: outer-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -223,7 +185,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -315,62 +277,7 @@ spec: metadata: {} name: comp-condition-3 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - outer-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -400,6 +307,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: outer-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/if_else_with_oneof_parameters.yaml b/test_data/compiled-workflows/if_else_with_oneof_parameters.yaml index f6f8db86b49..a1670b09c48 100644 --- a/test_data/compiled-workflows/if_else_with_oneof_parameters.yaml +++ b/test_data/compiled-workflows/if_else_with_oneof_parameters.yaml @@ -53,73 +53,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - flip-coin-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -137,16 +71,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: flip-coin-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -209,7 +171,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -303,62 +265,7 @@ spec: metadata: {} name: comp-condition-3 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - flip-coin-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -388,6 +295,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: flip-coin-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/input_artifact.yaml b/test_data/compiled-workflows/input_artifact.yaml index b5b04b2cb65..54354700ae5 100644 --- a/test_data/compiled-workflows/input_artifact.yaml +++ b/test_data/compiled-workflows/input_artifact.yaml @@ -29,73 +29,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - input-artifact - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -113,16 +47,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: input-artifact + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -185,7 +147,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -245,62 +207,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - input-artifact - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -330,6 +237,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: input-artifact + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/iris_pipeline_compiled.yaml b/test_data/compiled-workflows/iris_pipeline_compiled.yaml index 4092416d52b..479f4f15b5e 100644 --- a/test_data/compiled-workflows/iris_pipeline_compiled.yaml +++ b/test_data/compiled-workflows/iris_pipeline_compiled.yaml @@ -82,73 +82,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - iris-training-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -166,16 +100,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: iris-training-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -238,7 +200,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -348,62 +310,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - iris-training-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -433,6 +340,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: iris-training-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/lightweight_python_functions_pipeline.yaml b/test_data/compiled-workflows/lightweight_python_functions_pipeline.yaml index cd9e9906024..64a68aa9208 100644 --- a/test_data/compiled-workflows/lightweight_python_functions_pipeline.yaml +++ b/test_data/compiled-workflows/lightweight_python_functions_pipeline.yaml @@ -84,73 +84,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - my-test-pipeline-beta - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -168,16 +102,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: my-test-pipeline-beta + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -240,7 +202,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -325,62 +287,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - my-test-pipeline-beta - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -410,6 +317,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: my-test-pipeline-beta + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/lightweight_python_functions_with_outputs.yaml b/test_data/compiled-workflows/lightweight_python_functions_with_outputs.yaml index 9af03fe2546..3398b654e3f 100644 --- a/test_data/compiled-workflows/lightweight_python_functions_with_outputs.yaml +++ b/test_data/compiled-workflows/lightweight_python_functions_with_outputs.yaml @@ -76,73 +76,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - functions-with-outputs - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -160,16 +94,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: functions-with-outputs + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -232,7 +194,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -366,62 +328,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - functions-with-outputs - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -451,6 +358,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: functions-with-outputs + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/log_streaming_compiled.yaml b/test_data/compiled-workflows/log_streaming_compiled.yaml index 51b6f899e7a..666408a637c 100644 --- a/test_data/compiled-workflows/log_streaming_compiled.yaml +++ b/test_data/compiled-workflows/log_streaming_compiled.yaml @@ -33,73 +33,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - log-streaming-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -117,16 +51,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: log-streaming-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -189,7 +151,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -249,62 +211,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - log-streaming-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -334,6 +241,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: log-streaming-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/long-running.yaml b/test_data/compiled-workflows/long-running.yaml index ad508914429..893ba8194ac 100644 --- a/test_data/compiled-workflows/long-running.yaml +++ b/test_data/compiled-workflows/long-running.yaml @@ -20,73 +20,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - wait-awhile - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -104,16 +38,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: wait-awhile + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -176,7 +138,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -261,62 +223,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - wait-awhile - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -346,6 +253,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: wait-awhile + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/loop_consume_upstream.yaml b/test_data/compiled-workflows/loop_consume_upstream.yaml index 68a6ea7b0fc..188878488bf 100644 --- a/test_data/compiled-workflows/loop_consume_upstream.yaml +++ b/test_data/compiled-workflows/loop_consume_upstream.yaml @@ -77,73 +77,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - loop-consume-upstream - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -161,16 +95,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: loop-consume-upstream + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -233,7 +195,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -320,62 +282,7 @@ spec: metadata: {} name: comp-for-loop-1 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - loop-consume-upstream - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -405,6 +312,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: loop-consume-upstream + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/metrics_visualization_v2.yaml b/test_data/compiled-workflows/metrics_visualization_v2.yaml index 6e74ffe8eff..0488064f133 100644 --- a/test_data/compiled-workflows/metrics_visualization_v2.yaml +++ b/test_data/compiled-workflows/metrics_visualization_v2.yaml @@ -121,73 +121,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - metrics-visualization-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -205,16 +139,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: metrics-visualization-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -277,7 +239,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -433,62 +395,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - metrics-visualization-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -518,6 +425,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: metrics-visualization-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/missing_kubernetes_optional_inputs.yaml b/test_data/compiled-workflows/missing_kubernetes_optional_inputs.yaml index da9bc020eb0..7b959e66c29 100644 --- a/test_data/compiled-workflows/missing_kubernetes_optional_inputs.yaml +++ b/test_data/compiled-workflows/missing_kubernetes_optional_inputs.yaml @@ -32,73 +32,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - missing-kubernetes-optional-inputs-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -116,16 +50,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: missing-kubernetes-optional-inputs-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -188,7 +150,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -251,62 +213,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - missing-kubernetes-optional-inputs-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -336,6 +243,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: missing-kubernetes-optional-inputs-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/mixed_parameters.yaml b/test_data/compiled-workflows/mixed_parameters.yaml index 95f38757554..85de6b4d77d 100644 --- a/test_data/compiled-workflows/mixed_parameters.yaml +++ b/test_data/compiled-workflows/mixed_parameters.yaml @@ -46,73 +46,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - mixed-parameters-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -130,16 +64,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: mixed-parameters-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -202,7 +164,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -262,62 +224,7 @@ spec: metadata: {} name: comp-core outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - mixed-parameters-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -347,6 +254,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: mixed-parameters-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/modelcar.yaml b/test_data/compiled-workflows/modelcar.yaml index 9612302b372..ec222cff1c8 100644 --- a/test_data/compiled-workflows/modelcar.yaml +++ b/test_data/compiled-workflows/modelcar.yaml @@ -54,73 +54,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-modelcar-model - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -138,16 +72,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-modelcar-model + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -210,7 +172,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -364,62 +326,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-modelcar-model - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -449,6 +356,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-modelcar-model + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/mounted_cabundle_configmap.yaml b/test_data/compiled-workflows/mounted_cabundle_configmap.yaml index b1ad7a2bec9..27834dfe36b 100644 --- a/test_data/compiled-workflows/mounted_cabundle_configmap.yaml +++ b/test_data/compiled-workflows/mounted_cabundle_configmap.yaml @@ -20,78 +20,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - - --ca_cert_path - - /kfp/certs/ca.crt - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - volumeMounts: - - mountPath: /kfp/certs - name: custom-ca - inputs: + - inputs: parameters: - name: component - name: task @@ -109,23 +38,45 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition - volumes: - - configMap: - items: - - key: test-configmap-key - path: ca.crt - name: test-configmap-name - name: custom-ca + jsonPath: $.condition + plugin: + driver-plugin: + args: + ca_cert_path: /etc/ssl/certs/ca-certificates/ca.crt + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: echo + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -190,7 +141,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -256,67 +207,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - - --ca_cert_path - - /kfp/certs/ca.crt - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - volumeMounts: - - mountPath: /kfp/certs - name: custom-ca - inputs: + - inputs: parameters: - name: component - default: "" @@ -346,13 +237,34 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition - volumes: - - configMap: - items: - - key: test-configmap-key - path: ca.crt - name: test-configmap-name - name: custom-ca + plugin: + driver-plugin: + args: + ca_cert_path: /etc/ssl/certs/ca-certificates/ca.crt + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: echo + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/mounted_cabundle_secret.yaml b/test_data/compiled-workflows/mounted_cabundle_secret.yaml index eb653a98201..f78774d57a7 100644 --- a/test_data/compiled-workflows/mounted_cabundle_secret.yaml +++ b/test_data/compiled-workflows/mounted_cabundle_secret.yaml @@ -20,78 +20,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - - --ca_cert_path - - /kfp/certs/ca.crt - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - volumeMounts: - - mountPath: /kfp/certs - name: custom-ca - inputs: + - inputs: parameters: - name: component - name: task @@ -109,23 +38,45 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition - volumes: - - name: custom-ca - secret: - items: - - key: ca.crt - path: ca.crt - secretName: test-secret-name + jsonPath: $.condition + plugin: + driver-plugin: + args: + ca_cert_path: /etc/ssl/certs/ca-certificates/ca.crt + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: echo + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -190,7 +141,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -256,67 +207,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - - --ca_cert_path - - /kfp/certs/ca.crt - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - volumeMounts: - - mountPath: /kfp/certs - name: custom-ca - inputs: + - inputs: parameters: - name: component - default: "" @@ -346,13 +237,34 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition - volumes: - - name: custom-ca - secret: - items: - - key: ca.crt - path: ca.crt - secretName: test-secret-name + plugin: + driver-plugin: + args: + ca_cert_path: /etc/ssl/certs/ca-certificates/ca.crt + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: echo + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/multiple_artifacts_namedtuple.yaml b/test_data/compiled-workflows/multiple_artifacts_namedtuple.yaml index 6511708d3d7..dead7d3dabd 100644 --- a/test_data/compiled-workflows/multiple_artifacts_namedtuple.yaml +++ b/test_data/compiled-workflows/multiple_artifacts_namedtuple.yaml @@ -50,73 +50,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - multiple-artifacts-namedtuple-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -134,16 +68,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: multiple-artifacts-namedtuple-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -206,7 +168,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -266,62 +228,7 @@ spec: metadata: {} name: comp-core outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - multiple-artifacts-namedtuple-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -351,6 +258,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: multiple-artifacts-namedtuple-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/multiple_parameters_namedtuple.yaml b/test_data/compiled-workflows/multiple_parameters_namedtuple.yaml index 0a1b515896a..974347920b0 100644 --- a/test_data/compiled-workflows/multiple_parameters_namedtuple.yaml +++ b/test_data/compiled-workflows/multiple_parameters_namedtuple.yaml @@ -49,73 +49,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - multiple-parameters-namedtuple-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -133,16 +67,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: multiple-parameters-namedtuple-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -205,7 +167,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -265,62 +227,7 @@ spec: metadata: {} name: comp-core outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - multiple-parameters-namedtuple-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -350,6 +257,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: multiple-parameters-namedtuple-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/nested_pipeline_opt_input_child_level_compiled.yaml b/test_data/compiled-workflows/nested_pipeline_opt_input_child_level_compiled.yaml index c7c072d2fb1..0a79b693e14 100644 --- a/test_data/compiled-workflows/nested_pipeline_opt_input_child_level_compiled.yaml +++ b/test_data/compiled-workflows/nested_pipeline_opt_input_child_level_compiled.yaml @@ -109,73 +109,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - nested-pipeline-opt-input-child-level - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -193,16 +127,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: nested-pipeline-opt-input-child-level + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -265,7 +227,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -445,62 +407,7 @@ spec: metadata: {} name: comp-nested-pipeline outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - nested-pipeline-opt-input-child-level - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -530,6 +437,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: nested-pipeline-opt-input-child-level + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml b/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml index bc9f441922b..29e4e1949d6 100644 --- a/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml +++ b/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml @@ -60,73 +60,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - nested-pipeline-opt-inputs-nil - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -144,16 +78,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: nested-pipeline-opt-inputs-nil + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -216,7 +178,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -324,62 +286,7 @@ spec: metadata: {} name: comp-nested-pipeline outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - nested-pipeline-opt-inputs-nil - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -409,6 +316,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: nested-pipeline-opt-inputs-nil + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/nested_pipeline_opt_inputs_parent_level_compiled.yaml b/test_data/compiled-workflows/nested_pipeline_opt_inputs_parent_level_compiled.yaml index 937a668ac47..41163c0f4f7 100644 --- a/test_data/compiled-workflows/nested_pipeline_opt_inputs_parent_level_compiled.yaml +++ b/test_data/compiled-workflows/nested_pipeline_opt_inputs_parent_level_compiled.yaml @@ -112,73 +112,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - nested-pipeline-opt-inputs-parent-level - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -196,16 +130,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: nested-pipeline-opt-inputs-parent-level + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -268,7 +230,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -456,62 +418,7 @@ spec: metadata: {} name: comp-nested-pipeline-non-nil-defaults outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - nested-pipeline-opt-inputs-parent-level - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -541,6 +448,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: nested-pipeline-opt-inputs-parent-level + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/nested_return.yaml b/test_data/compiled-workflows/nested_return.yaml index 941e29190b9..3c7c4ded5ab 100644 --- a/test_data/compiled-workflows/nested_return.yaml +++ b/test_data/compiled-workflows/nested_return.yaml @@ -30,73 +30,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - nested-return - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -114,16 +48,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: nested-return + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -186,7 +148,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -246,62 +208,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - nested-return - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -331,6 +238,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: nested-return + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/nested_with_parameters.yaml b/test_data/compiled-workflows/nested_with_parameters.yaml index d85eae4ca6a..25e6ec23585 100644 --- a/test_data/compiled-workflows/nested_with_parameters.yaml +++ b/test_data/compiled-workflows/nested_with_parameters.yaml @@ -62,73 +62,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -146,16 +80,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: math-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -218,7 +180,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -327,62 +289,7 @@ spec: metadata: {} name: comp-for-loop-4 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -412,6 +319,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: math-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/notebook_component_mixed.yaml b/test_data/compiled-workflows/notebook_component_mixed.yaml index cd3eb695357..b474cdff968 100644 --- a/test_data/compiled-workflows/notebook_component_mixed.yaml +++ b/test_data/compiled-workflows/notebook_component_mixed.yaml @@ -258,73 +258,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - nb-mixed - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -342,16 +276,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: nb-mixed + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -414,7 +376,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -524,62 +486,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - nb-mixed - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -609,6 +516,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: nb-mixed + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/notebook_component_simple.yaml b/test_data/compiled-workflows/notebook_component_simple.yaml index ede94fa5445..c6aaaf82fa8 100644 --- a/test_data/compiled-workflows/notebook_component_simple.yaml +++ b/test_data/compiled-workflows/notebook_component_simple.yaml @@ -127,73 +127,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - nb-simple - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -211,16 +145,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: nb-simple + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -283,7 +245,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -343,62 +305,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - nb-simple - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -428,6 +335,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: nb-simple + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/output_metrics.yaml b/test_data/compiled-workflows/output_metrics.yaml index 5521bd3d944..f11d3fe9041 100644 --- a/test_data/compiled-workflows/output_metrics.yaml +++ b/test_data/compiled-workflows/output_metrics.yaml @@ -31,73 +31,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - output-metrics - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -115,16 +49,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: output-metrics + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -187,7 +149,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -247,62 +209,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - output-metrics - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -332,6 +239,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: output-metrics + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/parallel_for_after_dependency.yaml b/test_data/compiled-workflows/parallel_for_after_dependency.yaml index 4a1ac6e3f9e..5660c5da9a0 100644 --- a/test_data/compiled-workflows/parallel_for_after_dependency.yaml +++ b/test_data/compiled-workflows/parallel_for_after_dependency.yaml @@ -32,73 +32,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - loop-with-after-dependency-set - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -116,16 +50,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: loop-with-after-dependency-set + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -188,7 +150,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -248,62 +210,7 @@ spec: metadata: {} name: comp-for-loop-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - loop-with-after-dependency-set - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -333,6 +240,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: loop-with-after-dependency-set + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/parameter_cache.yaml b/test_data/compiled-workflows/parameter_cache.yaml index a5e9759991c..c1d0509dd24 100644 --- a/test_data/compiled-workflows/parameter_cache.yaml +++ b/test_data/compiled-workflows/parameter_cache.yaml @@ -46,73 +46,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - parameter-cache-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -130,16 +64,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: parameter-cache-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -202,7 +164,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -262,62 +224,7 @@ spec: metadata: {} name: comp-core outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - parameter-cache-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -347,6 +254,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: parameter-cache-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/parameter_oneof.yaml b/test_data/compiled-workflows/parameter_oneof.yaml index a37e91be033..dedfe7d7604 100644 --- a/test_data/compiled-workflows/parameter_oneof.yaml +++ b/test_data/compiled-workflows/parameter_oneof.yaml @@ -85,73 +85,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - parameter-oneof-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -169,16 +103,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: parameter-oneof-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -241,7 +203,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -335,62 +297,7 @@ spec: metadata: {} name: comp-condition-3 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - parameter-oneof-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -420,6 +327,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: parameter-oneof-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/parameters_complex.yaml b/test_data/compiled-workflows/parameters_complex.yaml index 1a7f1e5c14c..c70d767894e 100644 --- a/test_data/compiled-workflows/parameters_complex.yaml +++ b/test_data/compiled-workflows/parameters_complex.yaml @@ -79,73 +79,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -163,16 +97,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: math-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -235,7 +197,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -295,62 +257,7 @@ spec: metadata: {} name: comp-for-loop-4 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -380,6 +287,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: math-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/parameters_simple.yaml b/test_data/compiled-workflows/parameters_simple.yaml index 64f38fd79ed..67acd9b47c7 100644 --- a/test_data/compiled-workflows/parameters_simple.yaml +++ b/test_data/compiled-workflows/parameters_simple.yaml @@ -50,73 +50,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -134,16 +68,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: math-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -206,7 +168,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -266,62 +228,7 @@ spec: metadata: {} name: comp-for-loop-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -351,6 +258,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: math-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_as_exit_task.yaml b/test_data/compiled-workflows/pipeline_as_exit_task.yaml index bce16bc4015..fc08e3ce6f8 100644 --- a/test_data/compiled-workflows/pipeline_as_exit_task.yaml +++ b/test_data/compiled-workflows/pipeline_as_exit_task.yaml @@ -66,73 +66,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-task-final-status-conditional - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -150,16 +84,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-task-final-status-conditional + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -222,7 +184,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -283,62 +245,7 @@ spec: metadata: {} name: comp-condition-1 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-task-final-status-conditional - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -368,6 +275,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-task-final-status-conditional + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_in_pipeline.yaml b/test_data/compiled-workflows/pipeline_in_pipeline.yaml index 42868bffef0..903b2a1aacb 100644 --- a/test_data/compiled-workflows/pipeline_in_pipeline.yaml +++ b/test_data/compiled-workflows/pipeline_in_pipeline.yaml @@ -35,73 +35,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-in-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -119,16 +53,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-in-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -191,7 +153,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -276,62 +238,7 @@ spec: metadata: {} name: comp-inner-pipeline outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-in-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -361,6 +268,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-in-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml b/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml index 263b5d2cc4b..a09d11030ac 100644 --- a/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml +++ b/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml @@ -44,73 +44,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-in-pipeline-complex - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -128,16 +62,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-in-pipeline-complex + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -200,7 +162,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -292,62 +254,7 @@ spec: metadata: {} name: comp-condition-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-in-pipeline-complex - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -377,6 +284,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-in-pipeline-complex + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_in_pipeline_loaded_from_yaml.yaml b/test_data/compiled-workflows/pipeline_in_pipeline_loaded_from_yaml.yaml index 9c7db70e223..171f5e0eccd 100644 --- a/test_data/compiled-workflows/pipeline_in_pipeline_loaded_from_yaml.yaml +++ b/test_data/compiled-workflows/pipeline_in_pipeline_loaded_from_yaml.yaml @@ -51,73 +51,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-in-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -135,16 +69,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-in-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -207,7 +169,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -292,62 +254,7 @@ spec: metadata: {} name: comp-inner-pipeline outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-in-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -377,6 +284,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-in-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_producer_consumer.yaml b/test_data/compiled-workflows/pipeline_producer_consumer.yaml index e3f0fdca73d..471414e2931 100644 --- a/test_data/compiled-workflows/pipeline_producer_consumer.yaml +++ b/test_data/compiled-workflows/pipeline_producer_consumer.yaml @@ -82,73 +82,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -166,16 +100,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: math-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -238,7 +200,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -298,62 +260,7 @@ spec: metadata: {} name: comp-for-loop-2-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - math-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -383,6 +290,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: math-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_after.yaml b/test_data/compiled-workflows/pipeline_with_after.yaml index 1a62e629d7b..c2ba28efb62 100644 --- a/test_data/compiled-workflows/pipeline_with_after.yaml +++ b/test_data/compiled-workflows/pipeline_with_after.yaml @@ -23,73 +23,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-after - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -107,16 +41,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-after + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -179,7 +141,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -292,62 +254,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-after - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -377,6 +284,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-after + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_artifact_upload_download.yaml b/test_data/compiled-workflows/pipeline_with_artifact_upload_download.yaml index 1e622ecceab..54c3a3419b0 100644 --- a/test_data/compiled-workflows/pipeline_with_artifact_upload_download.yaml +++ b/test_data/compiled-workflows/pipeline_with_artifact_upload_download.yaml @@ -50,73 +50,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-datasets - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -134,16 +68,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-datasets + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -206,7 +168,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -291,62 +253,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-datasets - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -376,6 +283,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-datasets + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_concat_placeholder.yaml b/test_data/compiled-workflows/pipeline_with_concat_placeholder.yaml index c4b0f0fb029..9719d0b1753 100644 --- a/test_data/compiled-workflows/pipeline_with_concat_placeholder.yaml +++ b/test_data/compiled-workflows/pipeline_with_concat_placeholder.yaml @@ -22,73 +22,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - one-step-pipeline-with-concat-placeholder - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -106,16 +40,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: one-step-pipeline-with-concat-placeholder + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -178,7 +140,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -238,62 +200,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - one-step-pipeline-with-concat-placeholder - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -323,6 +230,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: one-step-pipeline-with-concat-placeholder + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_condition.yaml b/test_data/compiled-workflows/pipeline_with_condition.yaml index 471e270dd23..c25ada2c18c 100644 --- a/test_data/compiled-workflows/pipeline_with_condition.yaml +++ b/test_data/compiled-workflows/pipeline_with_condition.yaml @@ -48,73 +48,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - single-condition-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -132,16 +66,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: single-condition-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -204,7 +166,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -313,62 +275,7 @@ spec: metadata: {} name: comp-condition-1 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - single-condition-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -398,6 +305,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: single-condition-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml b/test_data/compiled-workflows/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml index 716e377db10..23ee85a61f4 100644 --- a/test_data/compiled-workflows/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml +++ b/test_data/compiled-workflows/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml @@ -145,73 +145,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-dynamic-condition-output - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -229,16 +163,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-dynamic-condition-output + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -301,7 +263,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -361,62 +323,7 @@ spec: metadata: {} name: comp-condition-1 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-dynamic-condition-output - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -446,6 +353,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-dynamic-condition-output + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_dynamic_importer_metadata.yaml b/test_data/compiled-workflows/pipeline_with_dynamic_importer_metadata.yaml index 35783a40ef4..1d65d0e6a1e 100644 --- a/test_data/compiled-workflows/pipeline_with_dynamic_importer_metadata.yaml +++ b/test_data/compiled-workflows/pipeline_with_dynamic_importer_metadata.yaml @@ -94,73 +94,7 @@ spec: metadata: {} name: system-importer outputs: {} - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-importer - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -178,16 +112,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-importer + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -250,7 +212,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -334,62 +296,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-importer - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -419,6 +326,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-importer + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_dynamic_task_output_custom_training_job.yaml b/test_data/compiled-workflows/pipeline_with_dynamic_task_output_custom_training_job.yaml index a5c0db72829..8cffe38bb92 100644 --- a/test_data/compiled-workflows/pipeline_with_dynamic_task_output_custom_training_job.yaml +++ b/test_data/compiled-workflows/pipeline_with_dynamic_task_output_custom_training_job.yaml @@ -107,73 +107,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -191,16 +125,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -263,7 +225,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -396,62 +358,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -481,6 +388,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_env.yaml b/test_data/compiled-workflows/pipeline_with_env.yaml index a597944863b..56c6a52fa63 100644 --- a/test_data/compiled-workflows/pipeline_with_env.yaml +++ b/test_data/compiled-workflows/pipeline_with_env.yaml @@ -35,73 +35,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-env - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -119,16 +53,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-env + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -191,7 +153,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -275,62 +237,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-env - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -360,6 +267,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-env + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_exit_handler.yaml b/test_data/compiled-workflows/pipeline_with_exit_handler.yaml index e3ac40122ea..b775ede6d65 100644 --- a/test_data/compiled-workflows/pipeline_with_exit_handler.yaml +++ b/test_data/compiled-workflows/pipeline_with_exit_handler.yaml @@ -47,73 +47,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-exit-handler - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -131,16 +65,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-exit-handler + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -203,7 +165,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -321,62 +283,7 @@ spec: metadata: {} name: exit-hook-root-print-op outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-exit-handler - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -406,6 +313,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-exit-handler + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_google_artifact_type.yaml b/test_data/compiled-workflows/pipeline_with_google_artifact_type.yaml index 1720711fa76..02a3e082d24 100644 --- a/test_data/compiled-workflows/pipeline_with_google_artifact_type.yaml +++ b/test_data/compiled-workflows/pipeline_with_google_artifact_type.yaml @@ -112,73 +112,7 @@ spec: metadata: {} name: system-importer outputs: {} - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-google-types - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -196,16 +130,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-google-types + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -268,7 +230,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -365,62 +327,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-google-types - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -450,6 +357,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-google-types + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_importer.yaml b/test_data/compiled-workflows/pipeline_with_importer.yaml index 532d0c6188e..4b8b899fe59 100644 --- a/test_data/compiled-workflows/pipeline_with_importer.yaml +++ b/test_data/compiled-workflows/pipeline_with_importer.yaml @@ -102,73 +102,7 @@ spec: metadata: {} name: system-importer outputs: {} - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-importer - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -186,16 +120,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-importer + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -258,7 +220,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -331,62 +293,7 @@ spec: metadata: {} name: comp-condition-1 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-importer - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -416,6 +323,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-importer + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_importer_and_gcpc_types.yaml b/test_data/compiled-workflows/pipeline_with_importer_and_gcpc_types.yaml index 248b6dd24b2..89723e6c36f 100644 --- a/test_data/compiled-workflows/pipeline_with_importer_and_gcpc_types.yaml +++ b/test_data/compiled-workflows/pipeline_with_importer_and_gcpc_types.yaml @@ -24,73 +24,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-importer-and-gcpc-type - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -108,16 +42,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-importer-and-gcpc-type + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -180,7 +142,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -310,62 +272,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-importer-and-gcpc-type - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -395,6 +302,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-importer-and-gcpc-type + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_importer_workspace.yaml b/test_data/compiled-workflows/pipeline_with_importer_workspace.yaml index dcd9616a752..ba35cd8d45c 100644 --- a/test_data/compiled-workflows/pipeline_with_importer_workspace.yaml +++ b/test_data/compiled-workflows/pipeline_with_importer_workspace.yaml @@ -176,73 +176,7 @@ spec: - name: kfp-workspace persistentVolumeClaim: claimName: '{{workflow.name}}-kfp-workspace' - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-importer-workspace - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -260,16 +194,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-importer-workspace + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -332,7 +294,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -442,62 +404,7 @@ spec: metadata: {} name: comp-import-stage outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-importer-workspace - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -527,6 +434,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-importer-workspace + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_input_status_state.yaml b/test_data/compiled-workflows/pipeline_with_input_status_state.yaml index 615c7109453..4c45fd09f1d 100644 --- a/test_data/compiled-workflows/pipeline_with_input_status_state.yaml +++ b/test_data/compiled-workflows/pipeline_with_input_status_state.yaml @@ -47,73 +47,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - status-state-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -131,16 +65,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: status-state-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -203,7 +165,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -295,62 +257,7 @@ spec: metadata: {} name: exit-hook-root-echo-state outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - status-state-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -380,6 +287,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: status-state-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_loops.yaml b/test_data/compiled-workflows/pipeline_with_loops.yaml index 7dca578ced8..b0c280c1c23 100644 --- a/test_data/compiled-workflows/pipeline_with_loops.yaml +++ b/test_data/compiled-workflows/pipeline_with_loops.yaml @@ -63,73 +63,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-loops - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -147,16 +81,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-loops + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -219,7 +181,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -439,62 +401,7 @@ spec: metadata: {} name: comp-for-loop-4 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-loops - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -524,6 +431,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-loops + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_loops_and_conditions.yaml b/test_data/compiled-workflows/pipeline_with_loops_and_conditions.yaml index b3189a5994f..61c5053bc03 100644 --- a/test_data/compiled-workflows/pipeline_with_loops_and_conditions.yaml +++ b/test_data/compiled-workflows/pipeline_with_loops_and_conditions.yaml @@ -109,73 +109,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-loops-and-conditions-multi-layers - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -193,16 +127,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-loops-and-conditions-multi-layers + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -265,7 +227,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -389,62 +351,7 @@ spec: metadata: {} name: comp-for-loop-7 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-loops-and-conditions-multi-layers - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -474,6 +381,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-loops-and-conditions-multi-layers + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_metadata_fields.yaml b/test_data/compiled-workflows/pipeline_with_metadata_fields.yaml index b2b2fab923f..bbb5888b6c3 100644 --- a/test_data/compiled-workflows/pipeline_with_metadata_fields.yaml +++ b/test_data/compiled-workflows/pipeline_with_metadata_fields.yaml @@ -60,73 +60,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - dataset-concatenator - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -144,16 +78,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: dataset-concatenator + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -216,7 +178,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -301,62 +263,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - dataset-concatenator - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -386,6 +293,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: dataset-concatenator + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_metrics_outputs.yaml b/test_data/compiled-workflows/pipeline_with_metrics_outputs.yaml index 703fffa644c..8fd715ef07f 100644 --- a/test_data/compiled-workflows/pipeline_with_metrics_outputs.yaml +++ b/test_data/compiled-workflows/pipeline_with_metrics_outputs.yaml @@ -34,73 +34,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-metrics-outputs - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -118,16 +52,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-metrics-outputs + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -190,7 +152,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -250,62 +212,7 @@ spec: metadata: {} name: comp-for-loop-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-metrics-outputs - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -335,6 +242,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-metrics-outputs + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_multiple_exit_handlers.yaml b/test_data/compiled-workflows/pipeline_with_multiple_exit_handlers.yaml index d4cd9c697f8..9b668c211a6 100644 --- a/test_data/compiled-workflows/pipeline_with_multiple_exit_handlers.yaml +++ b/test_data/compiled-workflows/pipeline_with_multiple_exit_handlers.yaml @@ -53,73 +53,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-multiple-exit-handlers - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -137,16 +71,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-multiple-exit-handlers + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -209,7 +171,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -457,62 +419,7 @@ spec: metadata: {} name: exit-hook-root-print-op-5 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-multiple-exit-handlers - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -542,6 +449,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-multiple-exit-handlers + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_nested_conditions.yaml b/test_data/compiled-workflows/pipeline_with_nested_conditions.yaml index 62041957342..e06399dbf85 100644 --- a/test_data/compiled-workflows/pipeline_with_nested_conditions.yaml +++ b/test_data/compiled-workflows/pipeline_with_nested_conditions.yaml @@ -50,73 +50,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - nested-conditions-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -134,16 +68,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: nested-conditions-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -206,7 +168,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -291,62 +253,7 @@ spec: metadata: {} name: comp-condition-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - nested-conditions-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -376,6 +283,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: nested-conditions-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_nested_conditions_yaml.yaml b/test_data/compiled-workflows/pipeline_with_nested_conditions_yaml.yaml index c2b021a104b..87e8481fb37 100644 --- a/test_data/compiled-workflows/pipeline_with_nested_conditions_yaml.yaml +++ b/test_data/compiled-workflows/pipeline_with_nested_conditions_yaml.yaml @@ -62,73 +62,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - conditional-execution-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -146,16 +80,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: conditional-execution-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -218,7 +180,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -314,62 +276,7 @@ spec: metadata: {} name: comp-condition-3 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - conditional-execution-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -399,6 +306,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: conditional-execution-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_nested_loops.yaml b/test_data/compiled-workflows/pipeline_with_nested_loops.yaml index a8a0666a4b7..47973a344f8 100644 --- a/test_data/compiled-workflows/pipeline_with_nested_loops.yaml +++ b/test_data/compiled-workflows/pipeline_with_nested_loops.yaml @@ -40,73 +40,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-nested-loops - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -124,16 +58,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-nested-loops + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -196,7 +158,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -256,62 +218,7 @@ spec: metadata: {} name: comp-for-loop-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-nested-loops - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -341,6 +248,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-nested-loops + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_only_display_name.yaml b/test_data/compiled-workflows/pipeline_with_only_display_name.yaml index 8d266854ee1..a448e2e9282 100644 --- a/test_data/compiled-workflows/pipeline_with_only_display_name.yaml +++ b/test_data/compiled-workflows/pipeline_with_only_display_name.yaml @@ -20,73 +20,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - echo-name - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -104,16 +38,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: echo-name + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -176,7 +138,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -236,62 +198,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - echo-name - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -321,6 +228,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: echo-name + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_outputs.yaml b/test_data/compiled-workflows/pipeline_with_outputs.yaml index cfe3aae749e..546f09b4dc4 100644 --- a/test_data/compiled-workflows/pipeline_with_outputs.yaml +++ b/test_data/compiled-workflows/pipeline_with_outputs.yaml @@ -36,73 +36,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-in-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -120,16 +54,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-in-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -192,7 +154,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -277,62 +239,7 @@ spec: metadata: {} name: comp-inner-pipeline outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-in-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -362,6 +269,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-in-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_parallelfor_parallelism.yaml b/test_data/compiled-workflows/pipeline_with_parallelfor_parallelism.yaml index e49048cd00d..512de22662f 100644 --- a/test_data/compiled-workflows/pipeline_with_parallelfor_parallelism.yaml +++ b/test_data/compiled-workflows/pipeline_with_parallelfor_parallelism.yaml @@ -127,73 +127,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-loops - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -211,16 +145,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-loops + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -283,7 +245,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -343,62 +305,7 @@ spec: metadata: {} name: comp-for-loop-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-loops - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -428,6 +335,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-loops + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_params_containing_format.yaml b/test_data/compiled-workflows/pipeline_with_params_containing_format.yaml index 070191fdefe..6a3d3bcda05 100644 --- a/test_data/compiled-workflows/pipeline_with_params_containing_format.yaml +++ b/test_data/compiled-workflows/pipeline_with_params_containing_format.yaml @@ -49,73 +49,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-pipelineparam-containing-format - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -133,16 +67,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-pipelineparam-containing-format + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -205,7 +167,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -266,62 +228,7 @@ spec: metadata: {} name: comp-for-loop-2 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-pipelineparam-containing-format - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -351,6 +258,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-pipelineparam-containing-format + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_placeholders.yaml b/test_data/compiled-workflows/pipeline_with_placeholders.yaml index d2ff8353ca9..d6bb902be12 100644 --- a/test_data/compiled-workflows/pipeline_with_placeholders.yaml +++ b/test_data/compiled-workflows/pipeline_with_placeholders.yaml @@ -36,73 +36,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-placeholders - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -120,16 +54,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-placeholders + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -192,7 +154,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -252,62 +214,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-placeholders - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -337,6 +244,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-placeholders + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_pod_metadata.yaml b/test_data/compiled-workflows/pipeline_with_pod_metadata.yaml index 4ac0f02b6f8..902083337d2 100644 --- a/test_data/compiled-workflows/pipeline_with_pod_metadata.yaml +++ b/test_data/compiled-workflows/pipeline_with_pod_metadata.yaml @@ -151,73 +151,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-pod-metadata - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -235,16 +169,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-pod-metadata + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -307,7 +269,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -415,7 +377,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -558,7 +520,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -683,7 +645,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -798,7 +760,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -1034,62 +996,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-pod-metadata - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -1119,6 +1026,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-pod-metadata + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_retry.yaml b/test_data/compiled-workflows/pipeline_with_retry.yaml index 89ee66bd859..dc5f6f295b9 100644 --- a/test_data/compiled-workflows/pipeline_with_retry.yaml +++ b/test_data/compiled-workflows/pipeline_with_retry.yaml @@ -29,73 +29,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - test-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -113,16 +47,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: test-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -201,7 +163,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -279,62 +241,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - test-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -364,6 +271,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: test-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_reused_component.yaml b/test_data/compiled-workflows/pipeline_with_reused_component.yaml index 2afa871dccb..45969af5ab6 100644 --- a/test_data/compiled-workflows/pipeline_with_reused_component.yaml +++ b/test_data/compiled-workflows/pipeline_with_reused_component.yaml @@ -29,73 +29,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-reused-component - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -113,16 +47,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-reused-component + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -185,7 +147,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -295,62 +257,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-reused-component - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -380,6 +287,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-reused-component + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml b/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml index 853cb05d268..60e484458d7 100644 --- a/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml +++ b/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml @@ -48,73 +48,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-secret-env - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -132,16 +66,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-secret-env + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -204,7 +166,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -291,62 +253,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-secret-env - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -376,6 +283,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-secret-env + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_secret_as_volume.yaml b/test_data/compiled-workflows/pipeline_with_secret_as_volume.yaml index 52d6c8f6929..2b65aa802f2 100644 --- a/test_data/compiled-workflows/pipeline_with_secret_as_volume.yaml +++ b/test_data/compiled-workflows/pipeline_with_secret_as_volume.yaml @@ -36,73 +36,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-secret-volume - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -120,16 +54,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-secret-volume + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -192,7 +154,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -254,62 +216,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-secret-volume - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -339,6 +246,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-secret-volume + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_string_machine_fields_pipeline_input.yaml b/test_data/compiled-workflows/pipeline_with_string_machine_fields_pipeline_input.yaml index 371d6cffef6..5b98555e739 100644 --- a/test_data/compiled-workflows/pipeline_with_string_machine_fields_pipeline_input.yaml +++ b/test_data/compiled-workflows/pipeline_with_string_machine_fields_pipeline_input.yaml @@ -29,73 +29,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -113,16 +47,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -185,7 +147,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -245,62 +207,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -330,6 +237,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml b/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml index c351bdb1a42..24192841038 100644 --- a/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml +++ b/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml @@ -72,7 +72,7 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef sum_numbers(a: int, b: int) -\u003e int:\n return a + b\n\n"],"image":"python:3.11","resources":{"accelerator":{"resourceCount":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}","resourceType":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"},"resourceCpuLimit":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}","resourceMemoryLimit":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}}' - name: components-root - value: '{"dag":{"tasks":{"accelerator-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-limit"},"taskInfo":{"name":"accelerator-limit"}},"accelerator-type":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-type"},"taskInfo":{"name":"accelerator-type"}},"cpu-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-cpu-limit"},"taskInfo":{"name":"cpu-limit"}},"memory-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-memory-limit"},"taskInfo":{"name":"memory-limit"}},"sum-numbers":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-limit","accelerator-type","cpu-limit","memory-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}}}}' + value: '{"dag":{"tasks":{"accelerator-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-limit"},"taskInfo":{"name":"accelerator-limit"}},"accelerator-type":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-type"},"taskInfo":{"name":"accelerator-type"}},"cpu-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-cpu-limit"},"taskInfo":{"name":"cpu-limit"}},"memory-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-memory-limit"},"taskInfo":{"name":"memory-limit"}},"sum-numbers":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-type","cpu-limit","memory-limit","accelerator-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}}}}' entrypoint: entrypoint podMetadata: annotations: @@ -81,73 +81,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -165,16 +99,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -237,7 +199,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -368,15 +330,15 @@ spec: - name: component value: '{{workflow.parameters.components-49f9a898b718a077f30b7fd8c02d39767cff91ff0bbda4379daf866a91dbdb1b}}' - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-limit","accelerator-type","cpu-limit","memory-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}' + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-type","cpu-limit","memory-limit","accelerator-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}' - name: container value: '{{workflow.parameters.implementations-49f9a898b718a077f30b7fd8c02d39767cff91ff0bbda4379daf866a91dbdb1b}}' - name: task-name value: sum-numbers - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - depends: accelerator-limit.Succeeded && accelerator-type.Succeeded && cpu-limit.Succeeded - && memory-limit.Succeeded + depends: accelerator-type.Succeeded && cpu-limit.Succeeded && memory-limit.Succeeded + && accelerator-limit.Succeeded name: sum-numbers-driver template: system-container-driver - arguments: @@ -395,62 +357,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -480,6 +387,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_submit_request.yaml b/test_data/compiled-workflows/pipeline_with_submit_request.yaml index d351e3f5a9a..6f509a4a96f 100644 --- a/test_data/compiled-workflows/pipeline_with_submit_request.yaml +++ b/test_data/compiled-workflows/pipeline_with_submit_request.yaml @@ -40,73 +40,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-external-request - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -124,16 +58,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-external-request + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -196,7 +158,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -280,62 +242,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-external-request - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -365,6 +272,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-external-request + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_task_final_status.yaml b/test_data/compiled-workflows/pipeline_with_task_final_status.yaml index 70722356510..0d90f158029 100644 --- a/test_data/compiled-workflows/pipeline_with_task_final_status.yaml +++ b/test_data/compiled-workflows/pipeline_with_task_final_status.yaml @@ -63,73 +63,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-task-final-status - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -147,16 +81,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-task-final-status + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -219,7 +181,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -336,62 +298,7 @@ spec: metadata: {} name: exit-hook-root-exit-op outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-task-final-status - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -421,6 +328,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-task-final-status + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_task_final_status_yaml.yaml b/test_data/compiled-workflows/pipeline_with_task_final_status_yaml.yaml index cb0a4468173..b494e7015fb 100644 --- a/test_data/compiled-workflows/pipeline_with_task_final_status_yaml.yaml +++ b/test_data/compiled-workflows/pipeline_with_task_final_status_yaml.yaml @@ -28,73 +28,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-task-final-status-yaml - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -112,16 +46,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-task-final-status-yaml + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -184,7 +146,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -276,62 +238,7 @@ spec: metadata: {} name: exit-hook-root-exit-op outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-task-final-status-yaml - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -361,6 +268,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-task-final-status-yaml + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_task_using_ignore_upstream_failure.yaml b/test_data/compiled-workflows/pipeline_with_task_using_ignore_upstream_failure.yaml index 5aa545d5e2d..3855f39c411 100644 --- a/test_data/compiled-workflows/pipeline_with_task_using_ignore_upstream_failure.yaml +++ b/test_data/compiled-workflows/pipeline_with_task_using_ignore_upstream_failure.yaml @@ -43,73 +43,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - my-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -127,16 +61,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: my-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -199,7 +161,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -298,62 +260,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - my-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -383,6 +290,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: my-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_utils.yaml b/test_data/compiled-workflows/pipeline_with_utils.yaml index 9b39b37abfb..edd22889735 100644 --- a/test_data/compiled-workflows/pipeline_with_utils.yaml +++ b/test_data/compiled-workflows/pipeline_with_utils.yaml @@ -33,73 +33,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-utils - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -117,16 +51,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-utils + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -189,7 +151,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -249,62 +211,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-utils - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -334,6 +241,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-utils + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_various_io_types.yaml b/test_data/compiled-workflows/pipeline_with_various_io_types.yaml index 65e14abc675..4644b8fd764 100644 --- a/test_data/compiled-workflows/pipeline_with_various_io_types.yaml +++ b/test_data/compiled-workflows/pipeline_with_various_io_types.yaml @@ -24,73 +24,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-various-types - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -108,16 +42,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-various-types + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -180,7 +142,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -265,62 +227,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-various-types - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -350,6 +257,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-various-types + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_volume.yaml b/test_data/compiled-workflows/pipeline_with_volume.yaml index eca1bd84462..c3a1b47250a 100644 --- a/test_data/compiled-workflows/pipeline_with_volume.yaml +++ b/test_data/compiled-workflows/pipeline_with_volume.yaml @@ -77,73 +77,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-volume - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -161,16 +95,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-volume + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -233,7 +195,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -352,62 +314,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-volume - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -437,6 +344,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-volume + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_volume_no_cache.yaml b/test_data/compiled-workflows/pipeline_with_volume_no_cache.yaml index 125bce359f5..ae5f8bcc57f 100644 --- a/test_data/compiled-workflows/pipeline_with_volume_no_cache.yaml +++ b/test_data/compiled-workflows/pipeline_with_volume_no_cache.yaml @@ -77,73 +77,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-volume-no-cache - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -161,16 +95,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-volume-no-cache + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -233,7 +195,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -352,62 +314,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-volume-no-cache - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -437,6 +344,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-volume-no-cache + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_workspace.yaml b/test_data/compiled-workflows/pipeline_with_workspace.yaml index cddb959e4ff..66e7339481e 100644 --- a/test_data/compiled-workflows/pipeline_with_workspace.yaml +++ b/test_data/compiled-workflows/pipeline_with_workspace.yaml @@ -53,73 +53,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-workspace - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -137,16 +71,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-workspace + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -209,7 +171,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -294,62 +256,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-workspace - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -379,6 +286,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-workspace + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/placeholder_with_if_placeholder_none_input_value.yaml b/test_data/compiled-workflows/placeholder_with_if_placeholder_none_input_value.yaml index e9d496fc9f4..93a1104219f 100644 --- a/test_data/compiled-workflows/placeholder_with_if_placeholder_none_input_value.yaml +++ b/test_data/compiled-workflows/placeholder_with_if_placeholder_none_input_value.yaml @@ -23,73 +23,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - one-step-pipeline-with-if-placeholder-supply-none - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -107,16 +41,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: one-step-pipeline-with-if-placeholder-supply-none + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -179,7 +141,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -239,62 +201,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - one-step-pipeline-with-if-placeholder-supply-none - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -324,6 +231,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: one-step-pipeline-with-if-placeholder-supply-none + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/preprocess.yaml b/test_data/compiled-workflows/preprocess.yaml index 10fca853dc5..4e6ea81351c 100644 --- a/test_data/compiled-workflows/preprocess.yaml +++ b/test_data/compiled-workflows/preprocess.yaml @@ -51,73 +51,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - preprocess - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -135,16 +69,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: preprocess + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -207,7 +169,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -267,62 +229,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - preprocess - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -352,6 +259,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: preprocess + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/producer_consumer_param_pipeline.yaml b/test_data/compiled-workflows/producer_consumer_param_pipeline.yaml index 259ffa29b40..c0278e3e534 100644 --- a/test_data/compiled-workflows/producer_consumer_param_pipeline.yaml +++ b/test_data/compiled-workflows/producer_consumer_param_pipeline.yaml @@ -27,73 +27,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - producer-consumer-param-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -111,16 +45,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: producer-consumer-param-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -183,7 +145,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -268,62 +230,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - producer-consumer-param-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -353,6 +260,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: producer-consumer-param-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pvc_mount.yaml b/test_data/compiled-workflows/pvc_mount.yaml index bcede40b7c5..565bd879543 100644 --- a/test_data/compiled-workflows/pvc_mount.yaml +++ b/test_data/compiled-workflows/pvc_mount.yaml @@ -47,73 +47,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pvc-mount-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -131,16 +65,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pvc-mount-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -203,7 +165,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -292,62 +254,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pvc-mount-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -377,6 +284,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pvc-mount-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml b/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml index cac8dd85918..35e54a94b6c 100644 --- a/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml +++ b/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml @@ -97,73 +97,7 @@ spec: metadata: {} name: system-importer outputs: {} - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - make-language-model-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -181,16 +115,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: make-language-model-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -253,7 +215,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -326,62 +288,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - make-language-model-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -411,6 +318,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: make-language-model-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pythonic_artifacts_test_pipeline.yaml b/test_data/compiled-workflows/pythonic_artifacts_test_pipeline.yaml index cfdd6da2967..edaac75d2ae 100644 --- a/test_data/compiled-workflows/pythonic_artifacts_test_pipeline.yaml +++ b/test_data/compiled-workflows/pythonic_artifacts_test_pipeline.yaml @@ -48,73 +48,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pythonic-artifacts-test - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -132,16 +66,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pythonic-artifacts-test + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -204,7 +166,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -289,62 +251,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pythonic-artifacts-test - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -374,6 +281,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pythonic-artifacts-test + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pythonic_artifacts_with_list_of_artifacts.yaml b/test_data/compiled-workflows/pythonic_artifacts_with_list_of_artifacts.yaml index 2411f1687dc..8a5b4a5c0d6 100644 --- a/test_data/compiled-workflows/pythonic_artifacts_with_list_of_artifacts.yaml +++ b/test_data/compiled-workflows/pythonic_artifacts_with_list_of_artifacts.yaml @@ -49,73 +49,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - make-and-join-datasets - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -133,16 +67,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: make-and-join-datasets + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -205,7 +167,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -265,62 +227,7 @@ spec: metadata: {} name: comp-for-loop-1 outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - make-and-join-datasets - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -350,6 +257,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: make-and-join-datasets + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pythonic_artifacts_with_multiple_returns.yaml b/test_data/compiled-workflows/pythonic_artifacts_with_multiple_returns.yaml index dddec14a6f0..df32472aab0 100644 --- a/test_data/compiled-workflows/pythonic_artifacts_with_multiple_returns.yaml +++ b/test_data/compiled-workflows/pythonic_artifacts_with_multiple_returns.yaml @@ -54,73 +54,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - split-datasets-and-return-first - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -138,16 +72,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: split-datasets-and-return-first + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -210,7 +172,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -270,62 +232,7 @@ spec: metadata: {} name: comp-splitter-pipeline outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - split-datasets-and-return-first - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -355,6 +262,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: split-datasets-and-return-first + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/ray_integration_compiled.yaml b/test_data/compiled-workflows/ray_integration_compiled.yaml index 5d7195424bd..a467221790f 100644 --- a/test_data/compiled-workflows/ray_integration_compiled.yaml +++ b/test_data/compiled-workflows/ray_integration_compiled.yaml @@ -94,73 +94,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - ray-integration-test - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -178,16 +112,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: ray-integration-test + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -250,7 +212,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -310,62 +272,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - ray-integration-test - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -395,6 +302,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: ray-integration-test + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/run_as_user_cache_disabled.yaml b/test_data/compiled-workflows/run_as_user_cache_disabled.yaml index 4af364a3391..49c2f423924 100644 --- a/test_data/compiled-workflows/run_as_user_cache_disabled.yaml +++ b/test_data/compiled-workflows/run_as_user_cache_disabled.yaml @@ -24,75 +24,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - - --log_level - - "3" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -110,16 +42,45 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "3" + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: echo + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -184,7 +145,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -244,64 +205,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - - --log_level - - "3" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -331,6 +235,34 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "3" + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: echo + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/run_as_user_cache_enabled.yaml b/test_data/compiled-workflows/run_as_user_cache_enabled.yaml index 4af364a3391..49c2f423924 100644 --- a/test_data/compiled-workflows/run_as_user_cache_enabled.yaml +++ b/test_data/compiled-workflows/run_as_user_cache_enabled.yaml @@ -24,75 +24,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - - --log_level - - "3" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -110,16 +42,45 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + log_level: "3" + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: echo + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -184,7 +145,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -244,64 +205,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - echo - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - - --log_level - - "3" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -331,6 +235,34 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + log_level: "3" + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: echo + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/sequential_v1.yaml b/test_data/compiled-workflows/sequential_v1.yaml index ea01855af8c..116e8a40c49 100644 --- a/test_data/compiled-workflows/sequential_v1.yaml +++ b/test_data/compiled-workflows/sequential_v1.yaml @@ -20,73 +20,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - sequential - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -104,16 +38,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: sequential + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -176,7 +138,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -260,62 +222,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - sequential - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -345,6 +252,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: sequential + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/sequential_v2.yaml b/test_data/compiled-workflows/sequential_v2.yaml index 9e4c0acce3a..1de49f9e028 100644 --- a/test_data/compiled-workflows/sequential_v2.yaml +++ b/test_data/compiled-workflows/sequential_v2.yaml @@ -24,73 +24,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - sequential - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -108,16 +42,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: sequential + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -180,7 +142,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -265,62 +227,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - sequential - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -350,6 +257,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: sequential + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/take_nap_compiled.yaml b/test_data/compiled-workflows/take_nap_compiled.yaml index f83a8ee3edf..5152d37a480 100644 --- a/test_data/compiled-workflows/take_nap_compiled.yaml +++ b/test_data/compiled-workflows/take_nap_compiled.yaml @@ -45,73 +45,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - take-nap-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -129,16 +63,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: take-nap-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -201,7 +163,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -286,62 +248,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - take-nap-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -371,6 +278,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: take-nap-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/take_nap_pipeline_root_compiled.yaml b/test_data/compiled-workflows/take_nap_pipeline_root_compiled.yaml index f83a8ee3edf..5152d37a480 100644 --- a/test_data/compiled-workflows/take_nap_pipeline_root_compiled.yaml +++ b/test_data/compiled-workflows/take_nap_pipeline_root_compiled.yaml @@ -45,73 +45,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - take-nap-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -129,16 +63,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: take-nap-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -201,7 +163,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -286,62 +248,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - take-nap-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -371,6 +278,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: take-nap-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/two_step_pipeline.yaml b/test_data/compiled-workflows/two_step_pipeline.yaml index 10cbeeb1180..6159ad7b8fa 100644 --- a/test_data/compiled-workflows/two_step_pipeline.yaml +++ b/test_data/compiled-workflows/two_step_pipeline.yaml @@ -25,73 +25,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - simple-two-step-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -109,16 +43,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: simple-two-step-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -181,7 +143,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -266,62 +228,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - simple-two-step-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -351,6 +258,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: simple-two-step-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/two_step_pipeline_containerized.yaml b/test_data/compiled-workflows/two_step_pipeline_containerized.yaml index dd9cd00470b..f8cc8cd32e5 100644 --- a/test_data/compiled-workflows/two_step_pipeline_containerized.yaml +++ b/test_data/compiled-workflows/two_step_pipeline_containerized.yaml @@ -26,73 +26,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - containerized-two-step-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -110,16 +44,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: containerized-two-step-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -182,7 +144,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -267,62 +229,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - containerized-two-step-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -352,6 +259,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: containerized-two-step-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/upload_download_compiled.yaml b/test_data/compiled-workflows/upload_download_compiled.yaml index 2f60d9239ac..c155980628a 100644 --- a/test_data/compiled-workflows/upload_download_compiled.yaml +++ b/test_data/compiled-workflows/upload_download_compiled.yaml @@ -84,73 +84,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - test-data-passing-pipeline-1 - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -168,16 +102,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: test-data-passing-pipeline-1 + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -240,7 +202,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -350,62 +312,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - test-data-passing-pipeline-1 - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -435,6 +342,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: test-data-passing-pipeline-1 + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/xgboost_sample_pipeline.yaml b/test_data/compiled-workflows/xgboost_sample_pipeline.yaml index 44760145680..afd01ca8d8e 100644 --- a/test_data/compiled-workflows/xgboost_sample_pipeline.yaml +++ b/test_data/compiled-workflows/xgboost_sample_pipeline.yaml @@ -288,73 +288,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - xgboost-sample-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -372,16 +306,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: xgboost-sample-pipeline + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -444,7 +406,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -680,62 +642,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - xgboost-sample-pipeline - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -765,6 +672,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: xgboost-sample-pipeline + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: From 8e7a95d9003f0ec2ecc24cf2b818918bfd3ccf37 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Mon, 2 Feb 2026 00:32:33 +0300 Subject: [PATCH 11/51] merge Signed-off-by: arpechenin --- .../tls-enabled/driver-plugin-cm-path.yaml | 25 +++ .../standalone/tls-enabled/kustomization.yaml | 2 +- .../legacy-v2-api-integration-tests.yml | 7 + backend/Makefile | 8 +- .../pipelines-profile-controller/sync.py | 15 ++ .../base-tls-certs/kfp-api-cert.yaml | 28 --- .../kustomization.yaml | 4 + .../patches/ml-pipeline-driver-plugin-cm.yaml | 27 +++ .../pipeline_with_artifact_custom_path.yaml | 188 ++++++------------ ...ith_string_machine_fields_task_output.yaml | 8 +- 10 files changed, 151 insertions(+), 161 deletions(-) create mode 100644 .github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml create mode 100644 manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-driver-plugin-cm.yaml diff --git a/.github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml b/.github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml new file mode 100644 index 00000000000..d0a3acc2b99 --- /dev/null +++ b/.github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ml-pipeline-driver-agent +data: + sidecar.container: | + name: driver-plugin + image: kind-registry:5000/driver + ports: + - containerPort: 8080 + resources: + limits: + cpu: "1" + memory: 1Gi + requests: + cpu: 250m + memory: 512Mi + securityContext: + runAsNonRoot: true + runAsUser: 65534 + volumeMounts: + - name: argo-workflows-agent-ca-certificates + mountPath: /etc/ssl/certs/ca-certificates + readOnly: true + diff --git a/.github/resources/manifests/standalone/tls-enabled/kustomization.yaml b/.github/resources/manifests/standalone/tls-enabled/kustomization.yaml index 8b5c80e2eff..551b163214a 100644 --- a/.github/resources/manifests/standalone/tls-enabled/kustomization.yaml +++ b/.github/resources/manifests/standalone/tls-enabled/kustomization.yaml @@ -46,7 +46,7 @@ patches: target: kind: Deployment name: ml-pipeline - - path: ../../base/driver-plugin-cm-path.yaml + - path: driver-plugin-cm-path.yaml target: kind: ConfigMap name: ml-pipeline-driver-agent diff --git a/.github/workflows/legacy-v2-api-integration-tests.yml b/.github/workflows/legacy-v2-api-integration-tests.yml index 0378fd2bae5..72a3ba35fa8 100644 --- a/.github/workflows/legacy-v2-api-integration-tests.yml +++ b/.github/workflows/legacy-v2-api-integration-tests.yml @@ -97,9 +97,16 @@ jobs: PIPELINE_STORE: ${{ matrix.pipeline_store }} continue-on-error: true + - name: Debug TLS cert secret + if: ${{ always() && matrix.pod_to_pod_tls_enabled }} + run: | + echo "=== Trying to get kfp-api-tls-cert secret ===" + kubectl get secret kfp-api-tls-cert -n kubeflow || echo "Secret not found" + - name: Collect failed logs if: ${{ steps.deploy.outcome != 'success' || steps.forward-mlmd-port.outcome != 'success' || steps.tests.outcome != 'success' }} run: | + kubectl get secret kfp-api-tls-cert -n kubeflow ./.github/resources/scripts/collect-logs.sh --ns kubeflow --output /tmp/tmp_pod_log.txt exit 1 diff --git a/backend/Makefile b/backend/Makefile index ddae2ea9e47..2ff90d995bb 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -95,7 +95,13 @@ kind-cluster-agnostic: kubectl wait --for condition=established --timeout=1m crd/applications.app.k8s.io # Deploy KFP @if [ "${TLS_ENABLED}" = "true" ]; then \ - kubectl apply -k $(CURDIR)/../manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls; \ + kubectl apply -k $(CURDIR)/../manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls; \ + echo "Waiting for certificate to be ready..."; \ + sleep 10; \ + echo "Copying secret..."; \ + kubectl -n kubeflow get secret kfp-api-tls-cert -o json | \ + jq 'del(.metadata.uid, .metadata.resourceVersion, .metadata.creationTimestamp, .metadata.managedFields) | .metadata.name = "argo-workflows-agent-ca-certificates"' | \ + kubectl apply -f -; \ else \ kubectl apply -k $(CURDIR)/../manifests/kustomize/env/platform-agnostic; \ fi diff --git a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py index 12fa8bd1926..396662d8a63 100644 --- a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py +++ b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py @@ -424,6 +424,21 @@ def sync(self, parent, attachments): "secretkey": base64.b64encode(s3_access_key["AccessKey"]["SecretAccessKey"].encode('utf-8')).decode("utf-8"), }, }) + desired_resources.append({ + # https://argo-workflows.readthedocs.io/en/latest/service-account-secrets/ + { + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": "default-editor.service-account-token", + "namespace": namespace, + "annotations": { + "kubernetes.io/service-account.name": "default-editor" + } + }, + "type": "kubernetes.io/service-account-token" + }, + }) return {"status": desired_status, "attachments": desired_resources} diff --git a/manifests/kustomize/env/cert-manager/base-tls-certs/kfp-api-cert.yaml b/manifests/kustomize/env/cert-manager/base-tls-certs/kfp-api-cert.yaml index f61d447f780..8dcdf1c623c 100644 --- a/manifests/kustomize/env/cert-manager/base-tls-certs/kfp-api-cert.yaml +++ b/manifests/kustomize/env/cert-manager/base-tls-certs/kfp-api-cert.yaml @@ -24,31 +24,3 @@ spec: kind: Issuer name: kfp-api-tls-selfsigned-issuer secretName: kfp-api-tls-cert ---- -# Necessary for mounting TLS CA certificate (created by cert-manager) to driver executor plugin -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: argo-workflows-agent-ca-certificates -spec: - commonName: argo-workflows-agent-ca-certificates - isCA: true - dnsNames: - - ml-pipeline - - ml-pipeline.kubeflow - - ml-pipeline.$(kfp-namespace).svc.cluster.local - - ml-pipeline-scheduledworkflow - - metadata-envoy - - metadata-envoy-service - - metadata-grpc-service - - metadata-grpc-service.kubeflow - - metadata-grpc-service.$(kfp-namespace).svc.cluster.local - # localhost included here because cert is used in KFP pod-to-pod TLS-enabled testing against localhost base URL - - localhost - ipAddresses: - # Necessary for running TLS-enabled cluster locally. - - 127.0.0.1 - issuerRef: - kind: Issuer - name: kfp-api-tls-selfsigned-issuer - secretName: argo-workflows-agent-ca-certificates \ No newline at end of file diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/kustomization.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/kustomization.yaml index dbd6a30d394..43397c3d2fa 100644 --- a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/kustomization.yaml +++ b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/kustomization.yaml @@ -42,6 +42,10 @@ patches: target: kind: Deployment name: ml-pipeline-persistenceagent + - path: patches/ml-pipeline-driver-plugin-cm.yaml + target: + kind: ConfigMap + name: ml-pipeline-driver-agent - path: patches/ml-pipeline-ui-deployment.yaml target: kind: Deployment diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-driver-plugin-cm.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-driver-plugin-cm.yaml new file mode 100644 index 00000000000..cdf75e8d197 --- /dev/null +++ b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-driver-plugin-cm.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ml-pipeline-driver-agent + labels: + workflows.argoproj.io/configmap-type: ExecutorPlugin +data: + sidecar.automountServiceAccountToken: "true" + sidecar.container: | + name: driver-plugin + image: ntny/kf-driver:aplha123 + ports: + - containerPort: 8080 + resources: + limits: + cpu: "1" + memory: 1Gi + requests: + cpu: 250m + memory: 512Mi + securityContext: + runAsNonRoot: true + runAsUser: 65534 + volumeMounts: + - name: argo-workflows-agent-ca-certificates + mountPath: /etc/ssl/certs/ca-certificates + readOnly: true diff --git a/test_data/compiled-workflows/pipeline_with_artifact_custom_path.yaml b/test_data/compiled-workflows/pipeline_with_artifact_custom_path.yaml index 4fabd3c4e71..cdface75e1f 100644 --- a/test_data/compiled-workflows/pipeline_with_artifact_custom_path.yaml +++ b/test_data/compiled-workflows/pipeline_with_artifact_custom_path.yaml @@ -51,73 +51,7 @@ spec: pipelines.kubeflow.org/v2_component: "true" serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - pipeline-with-custom-path-artifact - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - name: task @@ -145,6 +79,34 @@ spec: valueFrom: default: "true" jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-custom-path-artifact + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER - dag: tasks: - arguments: @@ -207,7 +169,7 @@ spec: resources: limits: cpu: 500m - memory: 128Mi + memory: 256Mi requests: cpu: 100m volumeMounts: @@ -292,62 +254,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - pipeline-with-custom-path-artifact - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - inputs: + - inputs: parameters: - name: component - default: "" @@ -368,15 +275,42 @@ spec: parameters: - name: execution-id valueFrom: - jsonPath: $.execution-id + path: /tmp/outputs/execution-id - name: iteration-count valueFrom: default: "0" - jsonPath: $.iteration-count + path: /tmp/outputs/iteration-count - name: condition valueFrom: default: "true" - jsonPath: $.condition + path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local + mlmd_server_port: "8887" + no_proxy: "" + pipeline_name: pipeline-with-custom-path-artifact + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml b/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml index 24192841038..0d24180f14c 100644 --- a/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml +++ b/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml @@ -72,7 +72,7 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef sum_numbers(a: int, b: int) -\u003e int:\n return a + b\n\n"],"image":"python:3.11","resources":{"accelerator":{"resourceCount":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}","resourceType":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"},"resourceCpuLimit":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}","resourceMemoryLimit":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}}' - name: components-root - value: '{"dag":{"tasks":{"accelerator-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-limit"},"taskInfo":{"name":"accelerator-limit"}},"accelerator-type":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-type"},"taskInfo":{"name":"accelerator-type"}},"cpu-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-cpu-limit"},"taskInfo":{"name":"cpu-limit"}},"memory-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-memory-limit"},"taskInfo":{"name":"memory-limit"}},"sum-numbers":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-type","cpu-limit","memory-limit","accelerator-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}}}}' + value: '{"dag":{"tasks":{"accelerator-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-limit"},"taskInfo":{"name":"accelerator-limit"}},"accelerator-type":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-type"},"taskInfo":{"name":"accelerator-type"}},"cpu-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-cpu-limit"},"taskInfo":{"name":"cpu-limit"}},"memory-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-memory-limit"},"taskInfo":{"name":"memory-limit"}},"sum-numbers":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-limit","accelerator-type","memory-limit","cpu-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}}}}' entrypoint: entrypoint podMetadata: annotations: @@ -330,15 +330,15 @@ spec: - name: component value: '{{workflow.parameters.components-49f9a898b718a077f30b7fd8c02d39767cff91ff0bbda4379daf866a91dbdb1b}}' - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-type","cpu-limit","memory-limit","accelerator-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}' + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-limit","accelerator-type","memory-limit","cpu-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}' - name: container value: '{{workflow.parameters.implementations-49f9a898b718a077f30b7fd8c02d39767cff91ff0bbda4379daf866a91dbdb1b}}' - name: task-name value: sum-numbers - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - depends: accelerator-type.Succeeded && cpu-limit.Succeeded && memory-limit.Succeeded - && accelerator-limit.Succeeded + depends: accelerator-limit.Succeeded && accelerator-type.Succeeded && memory-limit.Succeeded + && cpu-limit.Succeeded name: sum-numbers-driver template: system-container-driver - arguments: From 77d9f89c94d3a7632c5620260d6a44088803672d Mon Sep 17 00:00:00 2001 From: arpechenin Date: Wed, 4 Feb 2026 23:32:06 +0300 Subject: [PATCH 12/51] debug kfp-api-tls-cert creation and availability Signed-off-by: arpechenin --- .github/workflows/legacy-v2-api-integration-tests.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/legacy-v2-api-integration-tests.yml b/.github/workflows/legacy-v2-api-integration-tests.yml index 72a3ba35fa8..ecae6a2f059 100644 --- a/.github/workflows/legacy-v2-api-integration-tests.yml +++ b/.github/workflows/legacy-v2-api-integration-tests.yml @@ -87,6 +87,11 @@ jobs: run: kubectl -n kubeflow port-forward svc/metadata-grpc-service 8080:8080 & continue-on-error: true + - name: Debug TLS cert secret before tests + if: ${{ always() && matrix.pod_to_pod_tls_enabled }} + run: | + kubectl get secret kfp-api-tls-cert -n kubeflow + - name: API integration tests v2 id: tests if: ${{ steps.forward-mlmd-port.outcome == 'success' }} @@ -97,11 +102,10 @@ jobs: PIPELINE_STORE: ${{ matrix.pipeline_store }} continue-on-error: true - - name: Debug TLS cert secret + - name: Debug TLS cert secret after tests if: ${{ always() && matrix.pod_to_pod_tls_enabled }} run: | - echo "=== Trying to get kfp-api-tls-cert secret ===" - kubectl get secret kfp-api-tls-cert -n kubeflow || echo "Secret not found" + kubectl get secret kfp-api-tls-cert -n kubeflow - name: Collect failed logs if: ${{ steps.deploy.outcome != 'success' || steps.forward-mlmd-port.outcome != 'success' || steps.tests.outcome != 'success' }} From d19d099c3ea1992b48f7f57395544f0a9ca8300e Mon Sep 17 00:00:00 2001 From: arpechenin Date: Thu, 5 Feb 2026 23:03:27 +0300 Subject: [PATCH 13/51] add necessary sa to profile namespaces Signed-off-by: arpechenin --- .../pipelines-profile-controller/sync.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py index 396662d8a63..5c994cf7a91 100644 --- a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py +++ b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py @@ -426,6 +426,34 @@ def sync(self, parent, attachments): }) desired_resources.append({ # https://argo-workflows.readthedocs.io/en/latest/service-account-secrets/ + { + "apiVersion": "v1", + "kind": "ServiceAccount", + "metadata": { + "name": "ml-pipeline-driver-agent-executor-plugin", + "namespace": namespace, + "labels": { + "application-crd-id": "kubeflow-pipelines" + } + }, + "secrets": [ + { + "name": "ml-pipeline-driver-agent-executor-plugin.service-account-token" + } + ] + }, + { + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": "ml-pipeline-driver-agent-executor-plugin.service-account-token", + "namespace": namespace, + "annotations": { + "kubernetes.io/service-account.name": "ml-pipeline-driver-agent-executor-plugin" + } + }, + "type": "kubernetes.io/service-account-token" + }, { "apiVersion": "v1", "kind": "Secret", From 4d54eca9a31fd31d76aa8cf6c456c8cc60620ba9 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Fri, 6 Feb 2026 00:46:00 +0300 Subject: [PATCH 14/51] Fix failing driver task log capturing in e2e tests Signed-off-by: arpechenin --- backend/test/end2end/utils/e2e_utils.go | 16 +++++++++++++++- .../patches/ml-pipeline-driver-plugin-cm.yaml | 2 +- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/backend/test/end2end/utils/e2e_utils.go b/backend/test/end2end/utils/e2e_utils.go index 09526501700..a4bd2323361 100644 --- a/backend/test/end2end/utils/e2e_utils.go +++ b/backend/test/end2end/utils/e2e_utils.go @@ -5,6 +5,7 @@ import ( "fmt" "maps" "sort" + "strings" "time" runparams "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/run_client/run_service" @@ -121,11 +122,24 @@ func CapturePodLogsForUnsuccessfulTasks(k8Client *kubernetes.Clientset, testCont } case run_model.V2beta1RuntimeStateFAILED: { + agentLogsNotYetAcquired := true logger.Log("%s - Task %s for Run %s did not complete successfully", *task.State, task.DisplayName, task.RunID) for _, childTask := range task.ChildTasks { podName := childTask.PodName + isDriver := strings.Contains(task.DisplayName, "-driver") if podName != "" { - logger.Log("Capturing pod logs for task %s, with pod name %s", task.DisplayName, podName) + if isDriver { + if agentLogsNotYetAcquired { + logger.Log("Capturing pod logs for task executor agent, with pod name %s", podName) + podName += "-agent" + agentLogsNotYetAcquired = false + } else { + logger.Log("Logs from agent pod was already captured") + continue + } + } else { + logger.Log("Capturing pod logs for task %s, with pod name %s", task.DisplayName, podName) + } podLog := testutil.ReadPodLogs(k8Client, *config.Namespace, podName, nil, &testContext.TestStartTimeUTC, config.PodLogLimit) logger.Log("Pod logs captured for task %s in pod %s", task.DisplayName, podName) logger.Log("Attaching pod logs to the report") diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-driver-plugin-cm.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-driver-plugin-cm.yaml index cdf75e8d197..46f2b7a7e32 100644 --- a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-driver-plugin-cm.yaml +++ b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-driver-plugin-cm.yaml @@ -8,7 +8,7 @@ data: sidecar.automountServiceAccountToken: "true" sidecar.container: | name: driver-plugin - image: ntny/kf-driver:aplha123 + image: ghcr.io/kubeflow/kfp-driver:dummy ports: - containerPort: 8080 resources: From 5c7b636cfade986f6c2eba54f2d62058efb3d7f4 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Fri, 6 Feb 2026 22:48:12 +0300 Subject: [PATCH 15/51] - rework CA cert mounting - add cluster role for patching workflowtasksets/status in all namespaces Signed-off-by: arpechenin --- .../tls-enabled/driver-plugin-cm-path.yaml | 2 +- .github/workflows/api-server-tests.yml | 2 +- .github/workflows/e2e-test.yml | 2 +- .../legacy-v2-api-integration-tests.yml | 8 +++---- backend/Makefile | 6 ----- backend/src/apiserver/common/const.go | 4 ---- .../src/v2/compiler/argocompiler/container.go | 2 +- backend/src/v2/compiler/argocompiler/dag.go | 2 +- .../base-tls-certs/kfp-api-cert.yaml | 4 +++- .../patches/metadata-envoy-deployment.yaml | 2 +- .../patches/metadata-grpc-deployment.yaml | 2 +- .../patches/metadata-writer-deployment.yaml | 2 +- .../ml-pipeline-apiserver-deployment.yaml | 4 ++-- .../patches/ml-pipeline-driver-plugin-cm.yaml | 2 +- ...-pipeline-persistenceagent-deployment.yaml | 2 +- ...pipeline-scheduledworkflow-deployment.yaml | 2 +- .../patches/ml-pipeline-ui-deployment.yaml | 2 +- .../third-party/argo/base/kustomization.yaml | 1 + ...-controller-argo-taskset-clusterrole-.yaml | 24 +++++++++++++++++++ .../installs/namespace/kustomization.yaml | 11 +-------- .../mounted_cabundle_configmap.yaml | 4 ++-- .../mounted_cabundle_secret.yaml | 4 ++-- ...ith_string_machine_fields_task_output.yaml | 8 +++---- 23 files changed, 55 insertions(+), 47 deletions(-) create mode 100644 manifests/kustomize/third-party/argo/base/workflow-controller-argo-taskset-clusterrole-.yaml diff --git a/.github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml b/.github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml index d0a3acc2b99..ed43ddf7e6f 100644 --- a/.github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml +++ b/.github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml @@ -20,6 +20,6 @@ data: runAsUser: 65534 volumeMounts: - name: argo-workflows-agent-ca-certificates - mountPath: /etc/ssl/certs/ca-certificates + mountPath: /kfp/certs readOnly: true diff --git a/.github/workflows/api-server-tests.yml b/.github/workflows/api-server-tests.yml index 6cf86e91476..2135a3e29ba 100644 --- a/.github/workflows/api-server-tests.yml +++ b/.github/workflows/api-server-tests.yml @@ -111,7 +111,7 @@ jobs: shell: bash if: ${{ matrix.pod_to_pod_tls_enabled == 'true'}} run: | - kubectl get secret kfp-api-tls-cert -n kubeflow -o jsonpath='{.data.ca\.crt}' | base64 -d > "${{ github.workspace }}/ca.crt" + kubectl get secret argo-workflows-agent-ca-certificates -n kubeflow -o jsonpath='{.data.ca\.crt}' | base64 -d > "${{ github.workspace }}/ca.crt" echo "CA_CERT_PATH=${{ github.workspace }}/ca.crt" >> "$GITHUB_ENV" diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index c21bf3ee347..f280d56ba55 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -115,7 +115,7 @@ jobs: shell: bash if: ${{ matrix.pod_to_pod_tls_enabled == 'true'}} run: | - kubectl get secret kfp-api-tls-cert -n kubeflow -o jsonpath='{.data.ca\.crt}' | base64 -d > "${{ github.workspace }}/ca.crt" + kubectl get secret argo-workflows-agent-ca-certificates -n kubeflow -o jsonpath='{.data.ca\.crt}' | base64 -d > "${{ github.workspace }}/ca.crt" echo "CA_CERT_PATH=${{ github.workspace }}/ca.crt" >> "$GITHUB_ENV" - name: Configure Input Variables shell: bash diff --git a/.github/workflows/legacy-v2-api-integration-tests.yml b/.github/workflows/legacy-v2-api-integration-tests.yml index ecae6a2f059..f897e66ecf2 100644 --- a/.github/workflows/legacy-v2-api-integration-tests.yml +++ b/.github/workflows/legacy-v2-api-integration-tests.yml @@ -78,7 +78,7 @@ jobs: shell: bash if: ${{ matrix.pod_to_pod_tls_enabled }} run: | - kubectl get secret kfp-api-tls-cert -n kubeflow -o jsonpath='{.data.ca\.crt}' | base64 -d > "${{ github.workspace }}/ca.crt" + kubectl get secret argo-workflows-agent-ca-certificates -n kubeflow -o jsonpath='{.data.ca\.crt}' | base64 -d > "${{ github.workspace }}/ca.crt" echo "CA_CERT_PATH=${{ github.workspace }}/ca.crt" >> "$GITHUB_ENV" - name: Forward MLMD port @@ -90,7 +90,7 @@ jobs: - name: Debug TLS cert secret before tests if: ${{ always() && matrix.pod_to_pod_tls_enabled }} run: | - kubectl get secret kfp-api-tls-cert -n kubeflow + kubectl get secret argo-workflows-agent-ca-certificates -n kubeflow - name: API integration tests v2 id: tests @@ -105,12 +105,12 @@ jobs: - name: Debug TLS cert secret after tests if: ${{ always() && matrix.pod_to_pod_tls_enabled }} run: | - kubectl get secret kfp-api-tls-cert -n kubeflow + kubectl get secret argo-workflows-agent-ca-certificates-n kubeflow - name: Collect failed logs if: ${{ steps.deploy.outcome != 'success' || steps.forward-mlmd-port.outcome != 'success' || steps.tests.outcome != 'success' }} run: | - kubectl get secret kfp-api-tls-cert -n kubeflow + kubectl get secret argo-workflows-agent-ca-certificates -n kubeflow ./.github/resources/scripts/collect-logs.sh --ns kubeflow --output /tmp/tmp_pod_log.txt exit 1 diff --git a/backend/Makefile b/backend/Makefile index 2ff90d995bb..15386e71191 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -96,12 +96,6 @@ kind-cluster-agnostic: # Deploy KFP @if [ "${TLS_ENABLED}" = "true" ]; then \ kubectl apply -k $(CURDIR)/../manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls; \ - echo "Waiting for certificate to be ready..."; \ - sleep 10; \ - echo "Copying secret..."; \ - kubectl -n kubeflow get secret kfp-api-tls-cert -o json | \ - jq 'del(.metadata.uid, .metadata.resourceVersion, .metadata.creationTimestamp, .metadata.managedFields) | .metadata.name = "argo-workflows-agent-ca-certificates"' | \ - kubectl apply -f -; \ else \ kubectl apply -k $(CURDIR)/../manifests/kustomize/env/platform-agnostic; \ fi diff --git a/backend/src/apiserver/common/const.go b/backend/src/apiserver/common/const.go index e317c7fd233..584c8e63a01 100644 --- a/backend/src/apiserver/common/const.go +++ b/backend/src/apiserver/common/const.go @@ -69,10 +69,6 @@ const ( const ( CustomCaCertPath = "/kfp/certs/ca.crt" - // DriverCaCertPath is the path to the client CA certificate. - // The Driver, based on the Argo Workflow Executor Plugin, expects the CA certificate key - // to be mounted from the Secret at this path: /etc/ssl/certs/ca-certificates/ca.crt. - DriverCaCertPath = "/etc/ssl/certs/ca-certificates/ca.crt" CABundleDir = "/kfp/certs" ) diff --git a/backend/src/v2/compiler/argocompiler/container.go b/backend/src/v2/compiler/argocompiler/container.go index bb48698f971..2b6513d34f9 100644 --- a/backend/src/v2/compiler/argocompiler/container.go +++ b/backend/src/v2/compiler/argocompiler/container.go @@ -208,7 +208,7 @@ func (c *workflowCompiler) addContainerDriverTemplate() (string, error) { // If CABUNDLE_SECRET_NAME or CABUNDLE_CONFIGMAP_NAME is set, add ca_cert_path arg to container driver. if common.GetCaBundleSecretName() != "" || common.GetCaBundleConfigMapName() != "" { - args["ca_cert_path"] = common.DriverCaCertPath + args["ca_cert_path"] = common.CustomCaCertPath } if value, ok := os.LookupEnv(PipelineLogLevelEnvVar); ok { diff --git a/backend/src/v2/compiler/argocompiler/dag.go b/backend/src/v2/compiler/argocompiler/dag.go index 66104266fb4..d03e2c1e900 100644 --- a/backend/src/v2/compiler/argocompiler/dag.go +++ b/backend/src/v2/compiler/argocompiler/dag.go @@ -589,7 +589,7 @@ func (c *workflowCompiler) addDAGDriverTemplate() (string, error) { } // If CABUNDLE_SECRET_NAME add ca_cert_path arg to DAG driver. if common.GetCaBundleSecretName() != "" || common.GetCaBundleConfigMapName() != "" { - args["ca_cert_path"] = common.DriverCaCertPath + args["ca_cert_path"] = common.CustomCaCertPath } if value, ok := os.LookupEnv(PipelineLogLevelEnvVar); ok { diff --git a/manifests/kustomize/env/cert-manager/base-tls-certs/kfp-api-cert.yaml b/manifests/kustomize/env/cert-manager/base-tls-certs/kfp-api-cert.yaml index 8dcdf1c623c..47edb66303a 100644 --- a/manifests/kustomize/env/cert-manager/base-tls-certs/kfp-api-cert.yaml +++ b/manifests/kustomize/env/cert-manager/base-tls-certs/kfp-api-cert.yaml @@ -23,4 +23,6 @@ spec: issuerRef: kind: Issuer name: kfp-api-tls-selfsigned-issuer - secretName: kfp-api-tls-cert + # Unfortunately, Argo Workflows (for security reasons) can mount certificates + # into the agent only from a Secret with a predefined name `argo-workflows-agent-ca-certificates` + secretName: argo-workflows-agent-ca-certificates \ No newline at end of file diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-envoy-deployment.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-envoy-deployment.yaml index 0fa72587d37..c6a99880058 100644 --- a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-envoy-deployment.yaml +++ b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-envoy-deployment.yaml @@ -20,7 +20,7 @@ spec: volumes: - name: tls-certs secret: - secretName: kfp-api-tls-cert + secretName: argo-workflows-agent-ca-certificates - name: envoy-config-tls-enabled configMap: name: envoy-config diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-grpc-deployment.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-grpc-deployment.yaml index 4bc193483d7..e6af2e5f545 100644 --- a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-grpc-deployment.yaml +++ b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-grpc-deployment.yaml @@ -10,7 +10,7 @@ spec: volumes: - name: tls-certs secret: - secretName: kfp-api-tls-cert + secretName: argo-workflows-agent-ca-certificates - name: grpc-tls-config emptyDir: { } - name: mysql-secret diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-writer-deployment.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-writer-deployment.yaml index 1c651cc3b4b..f5f72c4b111 100644 --- a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-writer-deployment.yaml +++ b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/metadata-writer-deployment.yaml @@ -19,4 +19,4 @@ spec: volumes: - name: tls-certs secret: - secretName: kfp-api-tls-cert + secretName: argo-workflows-agent-ca-certificates diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-apiserver-deployment.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-apiserver-deployment.yaml index fef772368b5..59d268fe922 100644 --- a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-apiserver-deployment.yaml +++ b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-apiserver-deployment.yaml @@ -17,7 +17,7 @@ spec: - "--tlsCertKeyPath=/etc/pki/tls/certs/tls.key" env: - name: CABUNDLE_SECRET_NAME - value: "kfp-api-tls-cert" + value: "argo-workflows-agent-ca-certificates" - name: METADATA_TLS_ENABLED value: "true" - name: ML_PIPELINE_SERVICE_HOST @@ -53,4 +53,4 @@ spec: volumes: - name: tls-certs secret: - secretName: kfp-api-tls-cert + secretName: argo-workflows-agent-ca-certificates diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-driver-plugin-cm.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-driver-plugin-cm.yaml index 46f2b7a7e32..b8dfca5af76 100644 --- a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-driver-plugin-cm.yaml +++ b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-driver-plugin-cm.yaml @@ -23,5 +23,5 @@ data: runAsUser: 65534 volumeMounts: - name: argo-workflows-agent-ca-certificates - mountPath: /etc/ssl/certs/ca-certificates + mountPath: /kfp/certs readOnly: true diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-persistenceagent-deployment.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-persistenceagent-deployment.yaml index fccb9d2fedc..73f64d3ac06 100644 --- a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-persistenceagent-deployment.yaml +++ b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-persistenceagent-deployment.yaml @@ -35,4 +35,4 @@ spec: volumes: - name: tls-certs secret: - secretName: kfp-api-tls-cert + secretName: argo-workflows-agent-ca-certificates diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-scheduledworkflow-deployment.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-scheduledworkflow-deployment.yaml index 301ca3bb816..7c08cb7d4af 100644 --- a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-scheduledworkflow-deployment.yaml +++ b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-scheduledworkflow-deployment.yaml @@ -22,4 +22,4 @@ spec: volumes: - name: tls-certs secret: - secretName: kfp-api-tls-cert + secretName: argo-workflows-agent-ca-certificates diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-ui-deployment.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-ui-deployment.yaml index cf7775542b2..aed2004e31c 100644 --- a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-ui-deployment.yaml +++ b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-ui-deployment.yaml @@ -30,4 +30,4 @@ spec: volumes: - name: tls-certs secret: - secretName: kfp-api-tls-cert + secretName: argo-workflows-agent-ca-certificates diff --git a/manifests/kustomize/third-party/argo/base/kustomization.yaml b/manifests/kustomize/third-party/argo/base/kustomization.yaml index bddc0c22226..a342bd907b9 100644 --- a/manifests/kustomize/third-party/argo/base/kustomization.yaml +++ b/manifests/kustomize/third-party/argo/base/kustomization.yaml @@ -3,6 +3,7 @@ kind: Kustomization resources: - https://github.com/argoproj/argo-workflows/manifests/base/workflow-controller?ref=v3.7.3 +- workflow-controller-argo-taskset-clusterrole-.yaml patches: - path: workflow-controller-deployment-patch.yaml diff --git a/manifests/kustomize/third-party/argo/base/workflow-controller-argo-taskset-clusterrole-.yaml b/manifests/kustomize/third-party/argo/base/workflow-controller-argo-taskset-clusterrole-.yaml new file mode 100644 index 00000000000..61de55b59f7 --- /dev/null +++ b/manifests/kustomize/third-party/argo/base/workflow-controller-argo-taskset-clusterrole-.yaml @@ -0,0 +1,24 @@ +# Argo Workflow and the Agent communicate via the new WorkflowTaskSet CR, +# so RBAC permissions are required for this resource. +# Details: https://argo-workflows.readthedocs.io/en/latest/http-template/#argo-agent-rbac +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: argo-workflowtaskset-status +rules: + - apiGroups: ["argoproj.io"] + resources: ["workflowtasksets/status"] + verbs: ["get","list","watch","update","patch","delete","create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: argo-workflowtaskset-status-binding +subjects: + - kind: ServiceAccount + name: argo + namespace: kubeflow +roleRef: + kind: ClusterRole + name: argo-workflowtaskset-status + apiGroup: rbac.authorization.k8s.io diff --git a/manifests/kustomize/third-party/argo/installs/namespace/kustomization.yaml b/manifests/kustomize/third-party/argo/installs/namespace/kustomization.yaml index 28d36b235a3..cfb8c5b0595 100644 --- a/manifests/kustomize/third-party/argo/installs/namespace/kustomization.yaml +++ b/manifests/kustomize/third-party/argo/installs/namespace/kustomization.yaml @@ -15,13 +15,4 @@ patches: group: apps kind: Deployment name: workflow-controller - version: v1 -# Argo Workflow and the Agent communicate via the new WorkflowTaskSet CR, -# so RBAC permissions are required for this resource. -# Details: https://argo-workflows.readthedocs.io/en/latest/http-template/#argo-agent-rbac -- path: workflow-controller-argo-role-patch.json - target: - group: rbac.authorization.k8s.io - version: v1 - kind: Role - name: argo-role \ No newline at end of file + version: v1 \ No newline at end of file diff --git a/test_data/compiled-workflows/mounted_cabundle_configmap.yaml b/test_data/compiled-workflows/mounted_cabundle_configmap.yaml index 27834dfe36b..d3e8b33945d 100644 --- a/test_data/compiled-workflows/mounted_cabundle_configmap.yaml +++ b/test_data/compiled-workflows/mounted_cabundle_configmap.yaml @@ -51,7 +51,7 @@ spec: plugin: driver-plugin: args: - ca_cert_path: /etc/ssl/certs/ca-certificates/ca.crt + ca_cert_path: /kfp/certs/ca.crt cache_disabled: false cached_decision_path: '{{outputs.parameters.cached-decision.path}}' component: '{{inputs.parameters.component}}' @@ -240,7 +240,7 @@ spec: plugin: driver-plugin: args: - ca_cert_path: /etc/ssl/certs/ca-certificates/ca.crt + ca_cert_path: /kfp/certs/ca.crt cache_disabled: false component: '{{inputs.parameters.component}}' condition_path: '{{outputs.parameters.condition.path}}' diff --git a/test_data/compiled-workflows/mounted_cabundle_secret.yaml b/test_data/compiled-workflows/mounted_cabundle_secret.yaml index f78774d57a7..ecc0c6853ad 100644 --- a/test_data/compiled-workflows/mounted_cabundle_secret.yaml +++ b/test_data/compiled-workflows/mounted_cabundle_secret.yaml @@ -51,7 +51,7 @@ spec: plugin: driver-plugin: args: - ca_cert_path: /etc/ssl/certs/ca-certificates/ca.crt + ca_cert_path: /kfp/certs/ca.crt cache_disabled: false cached_decision_path: '{{outputs.parameters.cached-decision.path}}' component: '{{inputs.parameters.component}}' @@ -240,7 +240,7 @@ spec: plugin: driver-plugin: args: - ca_cert_path: /etc/ssl/certs/ca-certificates/ca.crt + ca_cert_path: /kfp/certs/ca.crt cache_disabled: false component: '{{inputs.parameters.component}}' condition_path: '{{outputs.parameters.condition.path}}' diff --git a/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml b/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml index 0d24180f14c..68e0552496f 100644 --- a/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml +++ b/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml @@ -72,7 +72,7 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef sum_numbers(a: int, b: int) -\u003e int:\n return a + b\n\n"],"image":"python:3.11","resources":{"accelerator":{"resourceCount":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}","resourceType":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"},"resourceCpuLimit":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}","resourceMemoryLimit":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}}' - name: components-root - value: '{"dag":{"tasks":{"accelerator-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-limit"},"taskInfo":{"name":"accelerator-limit"}},"accelerator-type":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-type"},"taskInfo":{"name":"accelerator-type"}},"cpu-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-cpu-limit"},"taskInfo":{"name":"cpu-limit"}},"memory-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-memory-limit"},"taskInfo":{"name":"memory-limit"}},"sum-numbers":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-limit","accelerator-type","memory-limit","cpu-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}}}}' + value: '{"dag":{"tasks":{"accelerator-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-limit"},"taskInfo":{"name":"accelerator-limit"}},"accelerator-type":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-type"},"taskInfo":{"name":"accelerator-type"}},"cpu-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-cpu-limit"},"taskInfo":{"name":"cpu-limit"}},"memory-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-memory-limit"},"taskInfo":{"name":"memory-limit"}},"sum-numbers":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-limit","accelerator-type","cpu-limit","memory-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}}}}' entrypoint: entrypoint podMetadata: annotations: @@ -330,15 +330,15 @@ spec: - name: component value: '{{workflow.parameters.components-49f9a898b718a077f30b7fd8c02d39767cff91ff0bbda4379daf866a91dbdb1b}}' - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-limit","accelerator-type","memory-limit","cpu-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}' + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-limit","accelerator-type","cpu-limit","memory-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}' - name: container value: '{{workflow.parameters.implementations-49f9a898b718a077f30b7fd8c02d39767cff91ff0bbda4379daf866a91dbdb1b}}' - name: task-name value: sum-numbers - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - depends: accelerator-limit.Succeeded && accelerator-type.Succeeded && memory-limit.Succeeded - && cpu-limit.Succeeded + depends: accelerator-limit.Succeeded && accelerator-type.Succeeded && cpu-limit.Succeeded + && memory-limit.Succeeded name: sum-numbers-driver template: system-container-driver - arguments: From 053499399995e94a9624af38594c4e49d129c8ad Mon Sep 17 00:00:00 2001 From: arpechenin Date: Sat, 7 Feb 2026 00:17:03 +0300 Subject: [PATCH 16/51] - add list events to e2e tests Signed-off-by: arpechenin --- backend/test/end2end/utils/e2e_utils.go | 6 +++++- backend/test/testutil/kubernetes_utils.go | 17 ++++++++++++++++- .../pipelines-profile-controller/sync.py | 7 ++++--- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/backend/test/end2end/utils/e2e_utils.go b/backend/test/end2end/utils/e2e_utils.go index a4bd2323361..a9213701b52 100644 --- a/backend/test/end2end/utils/e2e_utils.go +++ b/backend/test/end2end/utils/e2e_utils.go @@ -75,7 +75,11 @@ func ValidateComponentStatuses(runClient *apiserver.RunClient, k8Client *kuberne logger.Log("Updated pipeline run details") expectedTaskDetails := GetTasksFromWorkflow(compiledWorkflow) if *updatedRun.State == run_model.V2beta1RuntimeStateRUNNING { - logger.Log("Pipeline run did not finish, checking workflow controller logs") + logger.Log("Pipeline run did not finish") + logger.Log("Checking Argo WF events") + events := testutil.DescribeArgoWorkflow(k8Client, *config.Namespace) + ginkgo.AddReportEntry("Argo WF Events", events) + logger.Log("Checking workflow controller logs") podLog := testutil.ReadContainerLogs(k8Client, *config.Namespace, "workflow-controller", nil, &testContext.TestStartTimeUTC, config.PodLogLimit) logger.Log("Attaching Workflow Controller logs to the report") ginkgo.AddReportEntry("Workflow Controller Logs", podLog) diff --git a/backend/test/testutil/kubernetes_utils.go b/backend/test/testutil/kubernetes_utils.go index fa02c477d3c..f1ba4ba62b2 100644 --- a/backend/test/testutil/kubernetes_utils.go +++ b/backend/test/testutil/kubernetes_utils.go @@ -25,7 +25,6 @@ import ( "github.com/kubeflow/pipelines/backend/src/common/util" "github.com/kubeflow/pipelines/backend/test/logger" - "github.com/onsi/gomega" authenticationv1 "k8s.io/api/authentication/v1" v1 "k8s.io/api/core/v1" @@ -46,6 +45,22 @@ func CreateK8sClient() (*kubernetes.Clientset, error) { return k8sClient, nil } +func DescribeArgoWorkflow(client *kubernetes.Clientset, namespace string) string { + var allEvents []string + events, err := client.CoreV1().Events(namespace).List(context.TODO(), metav1.ListOptions{ + FieldSelector: "involvedObject.kind=Workflow", + }) + if err != nil { + logger.Log("failed to list Argo WF events: %v", err) + } + for _, e := range events.Items { + eventStr := fmt.Sprintf("- name: %s type: %s reason: %s message: %s lastTimestamp: %s", + e.Name, e.Type, e.Reason, e.Message, e.LastTimestamp) + allEvents = append(allEvents, eventStr) + } + return strings.Join(allEvents, ";\n") +} + // ReadContainerLogs - Read pod logs from a specific container func ReadContainerLogs(client *kubernetes.Clientset, namespace string, containerName string, follow *bool, sinceTime *time.Time, logLimit *int64) string { pod := GetPodContainingContainer(client, namespace, containerName) diff --git a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py index 5c994cf7a91..01ee09ef01a 100644 --- a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py +++ b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py @@ -424,7 +424,7 @@ def sync(self, parent, attachments): "secretkey": base64.b64encode(s3_access_key["AccessKey"]["SecretAccessKey"].encode('utf-8')).decode("utf-8"), }, }) - desired_resources.append({ + desired_resources.extend([ # https://argo-workflows.readthedocs.io/en/latest/service-account-secrets/ { "apiVersion": "v1", @@ -449,7 +449,8 @@ def sync(self, parent, attachments): "name": "ml-pipeline-driver-agent-executor-plugin.service-account-token", "namespace": namespace, "annotations": { - "kubernetes.io/service-account.name": "ml-pipeline-driver-agent-executor-plugin" + "kubernetes.io/service-account.name": + "ml-pipeline-driver-agent-executor-plugin" } }, "type": "kubernetes.io/service-account-token" @@ -466,7 +467,7 @@ def sync(self, parent, attachments): }, "type": "kubernetes.io/service-account-token" }, - }) + ]) return {"status": desired_status, "attachments": desired_resources} From 1feb4c30c2fef5bc7ed0eb08a6e36239cb37a2b3 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Sat, 7 Feb 2026 14:40:49 +0300 Subject: [PATCH 17/51] - create argo resources in end-user profiles Signed-off-by: arpechenin --- .../pipelines-profile-controller/sync.py | 74 +++++++++++++------ 1 file changed, 53 insertions(+), 21 deletions(-) diff --git a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py index 01ee09ef01a..8beacf7a087 100644 --- a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py +++ b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py @@ -253,7 +253,6 @@ def sync(self, parent, attachments): }, ] - # Add artifact fetcher related resources if enabled if artifacts_proxy_enabled.lower() == "true": desired_resources.extend([ { @@ -424,48 +423,81 @@ def sync(self, parent, attachments): "secretkey": base64.b64encode(s3_access_key["AccessKey"]["SecretAccessKey"].encode('utf-8')).decode("utf-8"), }, }) + + # Argo Workflow Executor Plugin Necessary Resources + agent_sa_name = "ml-pipeline-driver-agent-executor-plugin" + secret_name = f"{agent_sa_name}.service-account-token" + desired_resources.extend([ - # https://argo-workflows.readthedocs.io/en/latest/service-account-secrets/ { "apiVersion": "v1", - "kind": "ServiceAccount", + "kind": "Secret", "metadata": { - "name": "ml-pipeline-driver-agent-executor-plugin", + "name": "default-editor.service-account-token", "namespace": namespace, - "labels": { - "application-crd-id": "kubeflow-pipelines" + "annotations": { + "kubernetes.io/service-account.name": "default-editor" } }, - "secrets": [ - { - "name": "ml-pipeline-driver-agent-executor-plugin.service-account-token" - } - ] + "type": "kubernetes.io/service-account-token" }, { "apiVersion": "v1", - "kind": "Secret", + "kind": "ServiceAccount", "metadata": { - "name": "ml-pipeline-driver-agent-executor-plugin.service-account-token", + "name": agent_sa_name, "namespace": namespace, - "annotations": { - "kubernetes.io/service-account.name": - "ml-pipeline-driver-agent-executor-plugin" - } + "labels": { + "application-crd-id": "kubeflow-pipelines", + }, }, - "type": "kubernetes.io/service-account-token" }, { "apiVersion": "v1", "kind": "Secret", "metadata": { - "name": "default-editor.service-account-token", + "name": secret_name, "namespace": namespace, "annotations": { - "kubernetes.io/service-account.name": "default-editor" + "kubernetes.io/service-account.name": agent_sa_name, + }, + }, + "type": "kubernetes.io/service-account-token", + }, + { + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "Role", + "metadata": { + "name": "configmap-reader", + "namespace": namespace, + }, + "rules": [ + { + "apiGroups": [""], + "resources": ["configmaps"], + "verbs": ["get", "list", "watch"], } + ], + }, + { + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "RoleBinding", + "metadata": { + "name": "configmap-reader-binding", + "namespace": namespace, + }, + "subjects": [ + { + "kind": "ServiceAccount", + "name": agent_sa_name, + "namespace": namespace, + } + ], + "roleRef": { + "kind": "Role", + "name": "configmap-reader", + "apiGroup": "rbac.authorization.k8s.io", }, - "type": "kubernetes.io/service-account-token" }, ]) From 13d28a03d42a4e45529e4de6f641856990f87831 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Sat, 7 Feb 2026 15:10:24 +0300 Subject: [PATCH 18/51] - remove debugging tls in tests Signed-off-by: arpechenin --- .github/workflows/legacy-v2-api-integration-tests.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/legacy-v2-api-integration-tests.yml b/.github/workflows/legacy-v2-api-integration-tests.yml index f897e66ecf2..ca6b6191df4 100644 --- a/.github/workflows/legacy-v2-api-integration-tests.yml +++ b/.github/workflows/legacy-v2-api-integration-tests.yml @@ -102,11 +102,6 @@ jobs: PIPELINE_STORE: ${{ matrix.pipeline_store }} continue-on-error: true - - name: Debug TLS cert secret after tests - if: ${{ always() && matrix.pod_to_pod_tls_enabled }} - run: | - kubectl get secret argo-workflows-agent-ca-certificates-n kubeflow - - name: Collect failed logs if: ${{ steps.deploy.outcome != 'success' || steps.forward-mlmd-port.outcome != 'success' || steps.tests.outcome != 'success' }} run: | From d1261b559cf3bf5692803ae8a5a241f7331c0955 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Sat, 7 Feb 2026 19:01:37 +0300 Subject: [PATCH 19/51] - add debug logs to e2e Signed-off-by: arpechenin --- .github/workflows/e2e-test.yml | 5 +++++ backend/test/end2end/utils/e2e_utils.go | 7 +++++++ .../multi-user/pipelines-profile-controller/sync.py | 1 - 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index f280d56ba55..edca778e7af 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -236,6 +236,11 @@ jobs: run: | ./test/artifact-proxy/test-artifact-proxy.sh "$USER_NAMESPACE" + - name: Debug Created SA All Namespaces + if: ${{ always() }} + run: | + kubectl get sa --all-namespaces + - name: Run Tests uses: ./.github/actions/test-and-report if: ${{ steps.configure.outcome == 'success' }} diff --git a/backend/test/end2end/utils/e2e_utils.go b/backend/test/end2end/utils/e2e_utils.go index a9213701b52..0862a005517 100644 --- a/backend/test/end2end/utils/e2e_utils.go +++ b/backend/test/end2end/utils/e2e_utils.go @@ -88,6 +88,13 @@ func ValidateComponentStatuses(runClient *apiserver.RunClient, k8Client *kuberne } else { if *updatedRun.State != run_model.V2beta1RuntimeStateSUCCEEDED { logger.Log("Looks like the run %s FAILED, so capture pod logs for the failed task", runID) + logger.Log("Checking Argo WF events") + events := testutil.DescribeArgoWorkflow(k8Client, *config.Namespace) + ginkgo.AddReportEntry("Argo WF Events", events) + logger.Log("Checking workflow controller logs") + podLog := testutil.ReadContainerLogs(k8Client, *config.Namespace, "workflow-controller", nil, &testContext.TestStartTimeUTC, config.PodLogLimit) + logger.Log("Attaching Workflow Controller logs to the report") + ginkgo.AddReportEntry("Workflow Controller Logs", podLog) CapturePodLogsForUnsuccessfulTasks(k8Client, testContext, actualTaskDetails) ginkgo.Fail("Failing test because the pipeline run was not SUCCESSFUL") } else { diff --git a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py index 8beacf7a087..86fe066da76 100644 --- a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py +++ b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py @@ -427,7 +427,6 @@ def sync(self, parent, attachments): # Argo Workflow Executor Plugin Necessary Resources agent_sa_name = "ml-pipeline-driver-agent-executor-plugin" secret_name = f"{agent_sa_name}.service-account-token" - desired_resources.extend([ { "apiVersion": "v1", From 9793c72e2ea78b0a49f2ea4744ebf62e72884b8e Mon Sep 17 00:00:00 2001 From: arpechenin Date: Sat, 7 Feb 2026 19:55:16 +0300 Subject: [PATCH 20/51] - add debug logs to e2e Signed-off-by: arpechenin --- .github/resources/scripts/collect-logs.sh | 8 +++++++- backend/src/driver/main.go | 3 +++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/resources/scripts/collect-logs.sh b/.github/resources/scripts/collect-logs.sh index 89ea75324b1..77595793e83 100755 --- a/.github/resources/scripts/collect-logs.sh +++ b/.github/resources/scripts/collect-logs.sh @@ -52,7 +52,13 @@ function display_pod_info { kubectl describe pod "${POD_NAME}" -n "${NAMESPACE}" | grep -A 100 Events || echo "No events found for pod ${POD_NAME}." echo "----- LOGS -----" - kubectl logs "${POD_NAME}" -n "${NAMESPACE}" || echo "No logs found for pod ${POD_NAME}." + if [[ "${POD_NAME}" == *-agent* ]]; then + kubectl logs "${POD_NAME}" -n "${NAMESPACE}" -c driver-plugin || \ + echo "No logs found for pod ${POD_NAME}." + else + kubectl logs "${POD_NAME}" -n "${NAMESPACE}" || \ + echo "No logs found for pod ${POD_NAME}." + fi echo "===========================" echo "" diff --git a/backend/src/driver/main.go b/backend/src/driver/main.go index 7c1ab369818..900bfa94c55 100644 --- a/backend/src/driver/main.go +++ b/backend/src/driver/main.go @@ -21,6 +21,7 @@ import ( "flag" "fmt" "net/http" + "time" "google.golang.org/protobuf/encoding/protojson" @@ -74,6 +75,8 @@ func main() { if err != nil { glog.Warningf("Failed to start http server: %s", err.Error()) } + // temporary for preventing pod deletion before getting logs in e2e failed tests + time.Sleep(10 * time.Second) } // Use WARNING default logging level to facilitate troubleshooting. From 40aa133e06426087ce9f6fa78c3593be0a15d083 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Wed, 18 Feb 2026 19:17:03 +0300 Subject: [PATCH 21/51] update from master Signed-off-by: arpechenin --- backend/src/v2/cmd/driver/main.go | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 backend/src/v2/cmd/driver/main.go diff --git a/backend/src/v2/cmd/driver/main.go b/backend/src/v2/cmd/driver/main.go deleted file mode 100644 index e69de29bb2d..00000000000 From a630b383de5232c98b397dd2fc047b1ec4350f43 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Wed, 18 Feb 2026 19:34:17 +0300 Subject: [PATCH 22/51] update from master Signed-off-by: arpechenin --- backend/src/driver/main.go | 3 --- backend/src/driver/rpc_handler.go | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/backend/src/driver/main.go b/backend/src/driver/main.go index 900bfa94c55..7c1ab369818 100644 --- a/backend/src/driver/main.go +++ b/backend/src/driver/main.go @@ -21,7 +21,6 @@ import ( "flag" "fmt" "net/http" - "time" "google.golang.org/protobuf/encoding/protojson" @@ -75,8 +74,6 @@ func main() { if err != nil { glog.Warningf("Failed to start http server: %s", err.Error()) } - // temporary for preventing pod deletion before getting logs in e2e failed tests - time.Sleep(10 * time.Second) } // Use WARNING default logging level to facilitate troubleshooting. diff --git a/backend/src/driver/rpc_handler.go b/backend/src/driver/rpc_handler.go index 1b08e5d3409..08649e0db56 100644 --- a/backend/src/driver/rpc_handler.go +++ b/backend/src/driver/rpc_handler.go @@ -21,7 +21,9 @@ import ( "fmt" "io" "net/http" + "os" "strconv" + "strings" "github.com/golang/glog" "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" @@ -133,6 +135,18 @@ func drive(args api.DriverPluginArgs) (execution *driver.Execution, err error) { }() ctx := context.Background() + // Support reading component spec from a file if value starts with @ + // This bypasses exec() argument size limits for large workflows + if strings.HasPrefix(args.Component, "@") { + filePath := (args.Component)[1:] // Remove the "@" prefix + data, err := os.ReadFile(filePath) + if err != nil { + return nil, fmt.Errorf("failed to read component spec from file %s: %w", filePath, err) + } + args.Component = string(data) + glog.Infof("Read component spec from file: %s (%d bytes)", filePath, len(data)) + } + proxy.InitializeConfig(args.HTTPProxy, args.HTTPSProxy, args.NoProxy) glog.Infof("input ComponentSpec:%s\n", prettyPrint(args.Component)) From 77ead110675f90cc84f488630daec70c0a6049c3 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Sun, 22 Feb 2026 00:25:10 +0300 Subject: [PATCH 23/51] create executor-plugin SA, role per profile Signed-off-by: arpechenin --- .../deployment.yaml | 10 +- .../kustomization.yaml | 1 + .../pipelines-profile-controller-admin.yaml | 33 ++++ .../pipelines-profile-controller/sync.py | 174 ++++++++++-------- 4 files changed, 141 insertions(+), 77 deletions(-) create mode 100644 manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/pipelines-profile-controller-admin.yaml diff --git a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/deployment.yaml b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/deployment.yaml index 3652a67305f..6c348f015a3 100644 --- a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/deployment.yaml +++ b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/deployment.yaml @@ -9,6 +9,7 @@ spec: labels: sidecar.istio.io/inject: "false" spec: + serviceAccountName: pipelines-profile-controller-admin securityContext: runAsNonRoot: true seccompProfile: @@ -27,7 +28,14 @@ spec: type: RuntimeDefault # We just need an image with the python botocore library installed image: docker.io/alpine/k8s:1.32.3 - command: ["python", "/hooks/sync.py"] + command: + - sh + - -c + - | + python3 -m venv /tmp/venv && \ + . /tmp/venv/bin/activate && \ + pip install --no-cache-dir kubernetes boto3 && \ + python /hooks/sync.py envFrom: - configMapRef: name: kubeflow-pipelines-profile-controller-env diff --git a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/kustomization.yaml b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/kustomization.yaml index fdb0de23324..182a8a31619 100644 --- a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/kustomization.yaml +++ b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/kustomization.yaml @@ -5,6 +5,7 @@ resources: - service.yaml - deployment.yaml - decorator-controller.yaml +- pipelines-profile-controller-admin.yaml configMapGenerator: - files: - sync.py diff --git a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/pipelines-profile-controller-admin.yaml b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/pipelines-profile-controller-admin.yaml new file mode 100644 index 00000000000..1dc23beff6e --- /dev/null +++ b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/pipelines-profile-controller-admin.yaml @@ -0,0 +1,33 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: pipelines-profile-controller-admin + namespace: kubeflow +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: pipelines-profile-controller-admin-clusterrole +rules: + - apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: [""] + resources: ["roles"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["rbac.authorization.k8s.io"] + resources: ["rolebindings"] + verbs: ["create", "get", "list", "watch", "update", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: pipelines-profile-controller-admin-binding +subjects: + - kind: ServiceAccount + name: pipelines-profile-controller-admin + namespace: kubeflow +roleRef: + kind: ClusterRole + name: pipelines-profile-controller-admin-clusterrole + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py index 86fe066da76..6114638c0d9 100644 --- a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py +++ b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py @@ -17,6 +17,8 @@ import os import base64 import hashlib +from kubernetes import client, config +from kubernetes.client.rest import ApiException # From awscli installed in alpine/k8s image import botocore.session @@ -173,6 +175,68 @@ def upsert_lifecycle_policy(self, bucket_name, artifact_retention_days): else: print(f"ERROR: Failed to configure lifecycle policy: {exception}") + def upsert_executor_plugin_sa(self, namespace): + print('create executor plugin SAs for: ', namespace) + try: + config.load_incluster_config() + except: + config.load_kube_config() + core_v1 = client.CoreV1Api() + rbac_v1 = client.RbacAuthorizationV1Api() + agent_sa_name = "ml-pipeline-driver-agent-executor-plugin" + try: + core_v1.create_namespaced_service_account( + namespace=namespace, + body=client.V1ServiceAccount( + metadata=client.V1ObjectMeta( + name=agent_sa_name, + labels={"application-crd-id": "kubeflow-pipelines"}, + ) + ) + ) + print(f"ServiceAccount {agent_sa_name} created in {namespace}") + role_name = "configmap-reader" + role = client.V1Role( + metadata=client.V1ObjectMeta( + name=role_name, + namespace=namespace + ), + rules=[ + client.V1PolicyRule( + api_groups=[""], + resources=["configmaps"], + verbs=["get", "list", "watch"] + ) + ] + ) + rbac_v1.create_namespaced_role(namespace=namespace, body=role) + print(f"Role {role_name} created in {namespace}") + role_binding_name = "configmap-reader-binding" + role_binding = client.V1RoleBinding( + metadata=client.V1ObjectMeta( + name=role_binding_name, + namespace=namespace + ), + subjects=[client.V1Subject( + kind="ServiceAccount", + name=agent_sa_name, + namespace=namespace + )], + role_ref=client.V1RoleRef( + kind="Role", + name=role_name, + api_group="rbac.authorization.k8s.io" + ) + ) + rbac_v1.create_namespaced_role_binding(namespace=namespace, body=role_binding) + print(f"RoleBinding {role_binding_name} created in {namespace}") + except ApiException as e: + if e.status == 409: + print(f"ServiceAccount {agent_sa_name} already exists in {namespace}") + else: + print(f"Failed to create ServiceAccount {agent_sa_name}: {e}") + except Exception as e: + print(f"Unexpected error during the update sa in: {namespace} {e}") def sync(self, parent, attachments): # parent is a namespace @@ -194,6 +258,9 @@ def sync(self, parent, attachments): "True" or "False" } + + self.upsert_executor_plugin_sa(namespace) + # Generate the desired attachment object(s). desired_resources = [ { @@ -369,6 +436,37 @@ def sync(self, parent, attachments): }, ]) + print('Creating executor-plugin service accounts') + # Argo Workflow Executor Plugin Necessary Resources + agent_sa_name = "ml-pipeline-driver-agent-executor-plugin" + secret_name = f"{agent_sa_name}.service-account-token" + desired_resources.extend([ + { + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": "default-editor.service-account-token", + "namespace": namespace, + "annotations": { + "kubernetes.io/service-account.name": "default-editor" + } + }, + "type": "kubernetes.io/service-account-token" + }, + { + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": secret_name, + "namespace": namespace, + "annotations": { + "kubernetes.io/service-account.name": agent_sa_name, + }, + }, + "type": "kubernetes.io/service-account-token", + }, + ]) + print('Received request:\n', json.dumps(parent, sort_keys=True)) print('Desired resources except secrets:\n', json.dumps(desired_resources, sort_keys=True)) @@ -424,82 +522,6 @@ def sync(self, parent, attachments): }, }) - # Argo Workflow Executor Plugin Necessary Resources - agent_sa_name = "ml-pipeline-driver-agent-executor-plugin" - secret_name = f"{agent_sa_name}.service-account-token" - desired_resources.extend([ - { - "apiVersion": "v1", - "kind": "Secret", - "metadata": { - "name": "default-editor.service-account-token", - "namespace": namespace, - "annotations": { - "kubernetes.io/service-account.name": "default-editor" - } - }, - "type": "kubernetes.io/service-account-token" - }, - { - "apiVersion": "v1", - "kind": "ServiceAccount", - "metadata": { - "name": agent_sa_name, - "namespace": namespace, - "labels": { - "application-crd-id": "kubeflow-pipelines", - }, - }, - }, - { - "apiVersion": "v1", - "kind": "Secret", - "metadata": { - "name": secret_name, - "namespace": namespace, - "annotations": { - "kubernetes.io/service-account.name": agent_sa_name, - }, - }, - "type": "kubernetes.io/service-account-token", - }, - { - "apiVersion": "rbac.authorization.k8s.io/v1", - "kind": "Role", - "metadata": { - "name": "configmap-reader", - "namespace": namespace, - }, - "rules": [ - { - "apiGroups": [""], - "resources": ["configmaps"], - "verbs": ["get", "list", "watch"], - } - ], - }, - { - "apiVersion": "rbac.authorization.k8s.io/v1", - "kind": "RoleBinding", - "metadata": { - "name": "configmap-reader-binding", - "namespace": namespace, - }, - "subjects": [ - { - "kind": "ServiceAccount", - "name": agent_sa_name, - "namespace": namespace, - } - ], - "roleRef": { - "kind": "Role", - "name": "configmap-reader", - "apiGroup": "rbac.authorization.k8s.io", - }, - }, - ]) - return {"status": desired_status, "attachments": desired_resources} def do_POST(self): From d34b980d7175bf884676a517edf8bd4e048ced57 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Sun, 22 Feb 2026 00:31:19 +0300 Subject: [PATCH 24/51] create executor-plugin SA, role per profile Signed-off-by: arpechenin --- .../manifests/base/driver-plugin-cm-path.yaml | 3 ++- backend/src/driver/main.go | 4 +-- .../pipelines-profile-controller-admin.yaml | 8 +++--- .../pipelines-profile-controller/sync.py | 25 +++++++++++++------ ...-controller-argo-taskset-clusterrole-.yaml | 2 +- .../seaweedfs/kubeflow-edit-clusterrole.yaml | 3 +++ 6 files changed, 30 insertions(+), 15 deletions(-) diff --git a/.github/resources/manifests/base/driver-plugin-cm-path.yaml b/.github/resources/manifests/base/driver-plugin-cm-path.yaml index 54744ee94db..a2a32ba8c3f 100644 --- a/.github/resources/manifests/base/driver-plugin-cm-path.yaml +++ b/.github/resources/manifests/base/driver-plugin-cm-path.yaml @@ -5,7 +5,8 @@ metadata: data: sidecar.container: | name: driver-plugin - image: kind-registry:5000/driver + image: kind-registry:5000/driver:latest + imagePullPolicy: IfNotPresent ports: - containerPort: 8080 resources: diff --git a/backend/src/driver/main.go b/backend/src/driver/main.go index 7c1ab369818..4ebe6c21766 100644 --- a/backend/src/driver/main.go +++ b/backend/src/driver/main.go @@ -47,8 +47,8 @@ var ( logLevel = flag.String("log_level", "1", "The verbosity level to log.") // config - mlmdServerAddress = flag.String("mlmd_server_address", "metadata-grpc-service", "MLMD server address") - mlmdServerPort = flag.String("mlmd_server_port", "8080", "MLMD server port") + mlmdServerAddress = flag.String("mlmd_server_address", "", "MLMD server address") + mlmdServerPort = flag.String("mlmd_server_port", "", "MLMD server port") serverPort = flag.String("server_port", ":8080", "Server port") diff --git a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/pipelines-profile-controller-admin.yaml b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/pipelines-profile-controller-admin.yaml index 1dc23beff6e..8286abc1d5f 100644 --- a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/pipelines-profile-controller-admin.yaml +++ b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/pipelines-profile-controller-admin.yaml @@ -12,12 +12,12 @@ rules: - apiGroups: [""] resources: ["serviceaccounts"] verbs: ["create", "get", "list", "watch", "update", "delete"] - - apiGroups: [""] - resources: ["roles"] - verbs: ["create", "get", "list", "watch", "update", "delete"] - apiGroups: ["rbac.authorization.k8s.io"] - resources: ["rolebindings"] + resources: ["roles", "rolebindings"] verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: [ "" ] + resources: [ "configmaps" ] + verbs: [ "get", "list", "watch" ] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py index 6114638c0d9..ede1011c49b 100644 --- a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py +++ b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py @@ -211,24 +211,35 @@ def upsert_executor_plugin_sa(self, namespace): ) rbac_v1.create_namespaced_role(namespace=namespace, body=role) print(f"Role {role_name} created in {namespace}") + role_binding_name = "configmap-reader-binding" + namespace = 'kubeflow-user-example-com' + agent_sa_name = "ml-pipeline-driver-agent-executor-plugin" + role_name = "configmap-reader" role_binding = client.V1RoleBinding( metadata=client.V1ObjectMeta( - name=role_binding_name, + name="configmap-reader-binding", namespace=namespace ), - subjects=[client.V1Subject( - kind="ServiceAccount", - name=agent_sa_name, - namespace=namespace - )], + subjects=[ + { + "kind": "ServiceAccount", + "name": agent_sa_name, + "namespace": namespace, + "apiGroup": "" + } + ], role_ref=client.V1RoleRef( kind="Role", name=role_name, api_group="rbac.authorization.k8s.io" ) ) - rbac_v1.create_namespaced_role_binding(namespace=namespace, body=role_binding) + + rbac_v1.create_namespaced_role_binding( + namespace=namespace, + body=role_binding + ) print(f"RoleBinding {role_binding_name} created in {namespace}") except ApiException as e: if e.status == 409: diff --git a/manifests/kustomize/third-party/argo/base/workflow-controller-argo-taskset-clusterrole-.yaml b/manifests/kustomize/third-party/argo/base/workflow-controller-argo-taskset-clusterrole-.yaml index 61de55b59f7..18a0d260140 100644 --- a/manifests/kustomize/third-party/argo/base/workflow-controller-argo-taskset-clusterrole-.yaml +++ b/manifests/kustomize/third-party/argo/base/workflow-controller-argo-taskset-clusterrole-.yaml @@ -7,7 +7,7 @@ metadata: name: argo-workflowtaskset-status rules: - apiGroups: ["argoproj.io"] - resources: ["workflowtasksets/status"] + resources: ["workflowtasksets/status", "workflowtasksets"] verbs: ["get","list","watch","update","patch","delete","create"] --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/test_data/kubernetes/seaweedfs/kubeflow-edit-clusterrole.yaml b/test_data/kubernetes/seaweedfs/kubeflow-edit-clusterrole.yaml index bdf84eb8391..b27a925bf69 100644 --- a/test_data/kubernetes/seaweedfs/kubeflow-edit-clusterrole.yaml +++ b/test_data/kubernetes/seaweedfs/kubeflow-edit-clusterrole.yaml @@ -44,3 +44,6 @@ rules: - update - patch - delete +- apiGroups: ["argoproj.io"] + resources: ["workflowtasksets/status", "workflowtasksets"] + verbs: ["get","list","watch","update","patch","delete","create"] From a271f453127ca6fba7316d1366c9f4cf44126175 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Sun, 22 Feb 2026 17:46:33 +0300 Subject: [PATCH 25/51] - create executor-plugin SA, role per profile - fix mlmd connection parameters - add debug logs - disable istio-proxy injection for the agent pod Signed-off-by: arpechenin --- .github/resources/scripts/collect-logs.sh | 10 ++ .github/workflows/e2e-test.yml | 11 +- .../legacy-v2-api-integration-tests.yml | 6 - backend/src/driver/api/request.go | 2 + backend/src/driver/main.go | 14 +-- backend/src/driver/rpc_handler.go | 6 +- backend/src/v2/compiler/argocompiler/argo.go | 1 + .../src/v2/compiler/argocompiler/container.go | 10 +- backend/src/v2/compiler/argocompiler/dag.go | 5 +- backend/test/end2end/utils/e2e_utils.go | 6 - backend/test/testutil/kubernetes_utils.go | 16 --- test_data/compiled-workflows/add_numbers.yaml | 39 +++++- .../arguments_parameters.yaml | 39 +++++- .../compiled-workflows/artifact_cache.yaml | 39 +++++- .../compiled-workflows/artifact_crust.yaml | 39 +++++- .../compiled-workflows/artifacts_complex.yaml | 39 +++++- .../compiled-workflows/artifacts_simple.yaml | 39 +++++- .../collected_artifacts.yaml | 39 +++++- .../collected_parameters.yaml | 39 +++++- .../component_with_metadata_fields.yaml | 39 +++++- .../component_with_optional_inputs.yaml | 39 +++++- .../component_with_pip_index_urls.yaml | 39 +++++- .../component_with_pip_install.yaml | 39 +++++- .../component_with_pip_install_in_venv.yaml | 39 +++++- .../components_with_optional_artifacts.yaml | 51 +++++++- .../compiled-workflows/concat_message.yaml | 39 +++++- .../conditional_producer_and_consumers.yaml | 39 +++++- .../container_component_with_no_inputs.yaml | 39 +++++- .../compiled-workflows/container_io.yaml | 39 +++++- .../container_no_input.yaml | 39 +++++- .../container_with_artifact_output.yaml | 39 +++++- .../container_with_concat_placeholder.yaml | 39 +++++- .../container_with_if_placeholder.yaml | 39 +++++- ...container_with_placeholder_in_fstring.yaml | 39 +++++- .../containerized_python_component.yaml | 39 +++++- .../create_pod_metadata_complex.yaml | 75 +++++++++++- .../cross_loop_after_topology.yaml | 39 +++++- test_data/compiled-workflows/dict_input.yaml | 39 +++++- .../compiled-workflows/embedded_artifact.yaml | 39 +++++- test_data/compiled-workflows/env-var.yaml | 39 +++++- test_data/compiled-workflows/fail_v2.yaml | 39 +++++- test_data/compiled-workflows/flip_coin.yaml | 39 +++++- test_data/compiled-workflows/hello_world.yaml | 39 +++++- test_data/compiled-workflows/identity.yaml | 39 +++++- .../if_elif_else_complex.yaml | 39 +++++- .../if_elif_else_with_oneof_parameters.yaml | 39 +++++- .../if_else_with_oneof_artifacts.yaml | 39 +++++- .../if_else_with_oneof_parameters.yaml | 39 +++++- .../compiled-workflows/input_artifact.yaml | 39 +++++- .../iris_pipeline_compiled.yaml | 39 +++++- ...lightweight_python_functions_pipeline.yaml | 39 +++++- ...tweight_python_functions_with_outputs.yaml | 39 +++++- .../log_streaming_compiled.yaml | 39 +++++- .../compiled-workflows/long-running.yaml | 39 +++++- .../loop_consume_upstream.yaml | 39 +++++- .../metrics_visualization_v2.yaml | 39 +++++- .../missing_kubernetes_optional_inputs.yaml | 39 +++++- .../compiled-workflows/mixed_parameters.yaml | 39 +++++- test_data/compiled-workflows/modelcar.yaml | 51 +++++++- .../mounted_cabundle_configmap.yaml | 39 +++++- .../mounted_cabundle_secret.yaml | 39 +++++- .../multiple_artifacts_namedtuple.yaml | 39 +++++- .../multiple_parameters_namedtuple.yaml | 39 +++++- ...peline_opt_input_child_level_compiled.yaml | 39 +++++- ...sted_pipeline_opt_inputs_nil_compiled.yaml | 39 +++++- ...line_opt_inputs_parent_level_compiled.yaml | 39 +++++- .../compiled-workflows/nested_return.yaml | 39 +++++- .../nested_with_parameters.yaml | 39 +++++- .../notebook_component_mixed.yaml | 39 +++++- .../notebook_component_simple.yaml | 39 +++++- .../compiled-workflows/output_metrics.yaml | 39 +++++- .../parallel_for_after_dependency.yaml | 39 +++++- .../compiled-workflows/parameter_cache.yaml | 39 +++++- .../compiled-workflows/parameter_oneof.yaml | 39 +++++- .../parameters_complex.yaml | 39 +++++- .../compiled-workflows/parameters_simple.yaml | 39 +++++- .../pipeline_as_exit_task.yaml | 39 +++++- .../pipeline_in_pipeline.yaml | 39 +++++- .../pipeline_in_pipeline_complex.yaml | 39 +++++- ...pipeline_in_pipeline_loaded_from_yaml.yaml | 39 +++++- .../pipeline_producer_consumer.yaml | 39 +++++- .../pipeline_with_after.yaml | 39 +++++- .../pipeline_with_artifact_custom_path.yaml | 39 +++++- ...ipeline_with_artifact_upload_download.yaml | 39 +++++- .../pipeline_with_concat_placeholder.yaml | 39 +++++- .../pipeline_with_condition.yaml | 39 +++++- ...namic_task_output_custom_training_job.yaml | 39 +++++- ...peline_with_dynamic_importer_metadata.yaml | 51 +++++++- ...namic_task_output_custom_training_job.yaml | 39 +++++- .../compiled-workflows/pipeline_with_env.yaml | 39 +++++- .../pipeline_with_exit_handler.yaml | 39 +++++- .../pipeline_with_google_artifact_type.yaml | 51 +++++++- .../pipeline_with_importer.yaml | 51 +++++++- ...pipeline_with_importer_and_gcpc_types.yaml | 51 +++++++- .../pipeline_with_importer_workspace.yaml | 51 +++++++- .../pipeline_with_input_status_state.yaml | 39 +++++- .../pipeline_with_loops.yaml | 39 +++++- .../pipeline_with_loops_and_conditions.yaml | 39 +++++- .../pipeline_with_metadata_fields.yaml | 39 +++++- .../pipeline_with_metrics_outputs.yaml | 39 +++++- .../pipeline_with_multiple_exit_handlers.yaml | 39 +++++- .../pipeline_with_nested_conditions.yaml | 39 +++++- .../pipeline_with_nested_conditions_yaml.yaml | 39 +++++- .../pipeline_with_nested_loops.yaml | 39 +++++- .../pipeline_with_only_display_name.yaml | 39 +++++- .../pipeline_with_outputs.yaml | 39 +++++- ...pipeline_with_parallelfor_parallelism.yaml | 39 +++++- ...ipeline_with_params_containing_format.yaml | 39 +++++- .../pipeline_with_placeholders.yaml | 39 +++++- .../pipeline_with_pod_metadata.yaml | 111 +++++++++++++++++- .../pipeline_with_retry.yaml | 39 +++++- .../pipeline_with_reused_component.yaml | 39 +++++- .../pipeline_with_secret_as_env.yaml | 39 +++++- .../pipeline_with_secret_as_volume.yaml | 39 +++++- ..._string_machine_fields_pipeline_input.yaml | 39 +++++- ...ith_string_machine_fields_task_output.yaml | 39 +++++- .../pipeline_with_submit_request.yaml | 39 +++++- .../pipeline_with_task_final_status.yaml | 39 +++++- .../pipeline_with_task_final_status_yaml.yaml | 39 +++++- ...th_task_using_ignore_upstream_failure.yaml | 39 +++++- .../pipeline_with_utils.yaml | 39 +++++- .../pipeline_with_various_io_types.yaml | 39 +++++- .../pipeline_with_volume.yaml | 39 +++++- .../pipeline_with_volume_no_cache.yaml | 39 +++++- .../pipeline_with_workspace.yaml | 39 +++++- ..._with_if_placeholder_none_input_value.yaml | 39 +++++- test_data/compiled-workflows/preprocess.yaml | 39 +++++- .../producer_consumer_param_pipeline.yaml | 39 +++++- test_data/compiled-workflows/pvc_mount.yaml | 39 +++++- .../pythonic_artifact_with_single_return.yaml | 51 +++++++- .../pythonic_artifacts_test_pipeline.yaml | 39 +++++- ...onic_artifacts_with_list_of_artifacts.yaml | 39 +++++- ...honic_artifacts_with_multiple_returns.yaml | 39 +++++- .../ray_integration_compiled.yaml | 39 +++++- .../run_as_user_cache_disabled.yaml | 36 +++++- .../run_as_user_cache_enabled.yaml | 36 +++++- .../compiled-workflows/sequential_v1.yaml | 39 +++++- .../compiled-workflows/sequential_v2.yaml | 39 +++++- .../compiled-workflows/take_nap_compiled.yaml | 39 +++++- .../take_nap_pipeline_root_compiled.yaml | 39 +++++- .../compiled-workflows/two_step_pipeline.yaml | 39 +++++- .../two_step_pipeline_containerized.yaml | 39 +++++- .../upload_download_compiled.yaml | 39 +++++- .../xgboost_sample_pipeline.yaml | 39 +++++- 144 files changed, 4883 insertions(+), 589 deletions(-) diff --git a/.github/resources/scripts/collect-logs.sh b/.github/resources/scripts/collect-logs.sh index 77595793e83..95f9e0ea9f2 100755 --- a/.github/resources/scripts/collect-logs.sh +++ b/.github/resources/scripts/collect-logs.sh @@ -29,6 +29,15 @@ function check_namespace { return 0 } +function describe_argo_workflows { + local NAMESPACE=$1 + echo "===== Argo Workflows list =====" + kubectl describe wf -n "${NAMESPACE}" + echo "===== Argo Workflows data =====" + kubectl get events -n "${NAMESPACE}" --field-selector involvedObject.kind=Workflow --sort-by='.metadata.creationTimestamp' + echo "===============================" +} + function display_pod_info { local NAMESPACE=$1 @@ -70,6 +79,7 @@ function display_pod_info { if check_namespace "$NS"; then display_pod_info "$NS" + describe_argo_workflows "$NS" else exit 0 fi diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index edca778e7af..7ae220ae211 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -236,11 +236,6 @@ jobs: run: | ./test/artifact-proxy/test-artifact-proxy.sh "$USER_NAMESPACE" - - name: Debug Created SA All Namespaces - if: ${{ always() }} - run: | - kubectl get sa --all-namespaces - - name: Run Tests uses: ./.github/actions/test-and-report if: ${{ steps.configure.outcome == 'success' }} @@ -258,3 +253,9 @@ jobs: python_version: ${{ env.PYTHON_VERSION }} user_namespace: ${{ env.USER_NAMESPACE }} report_name: "E2EMultiUserTests_K8s=${{ matrix.k8s_version }}_cacheEnabled=${{ matrix.cache_enabled }}_multiUser=${{ matrix.multi_user }}" + + - name: Log Argo Wf data + if: ${{ always() }} + run: | + kubectl get wf --all-namespaces + kubectl get events --all-namespaces --field-selector involvedObject.kind=Workflow --sort-by='.metadata.creationTimestamp' diff --git a/.github/workflows/legacy-v2-api-integration-tests.yml b/.github/workflows/legacy-v2-api-integration-tests.yml index ca6b6191df4..41c4b4505a9 100644 --- a/.github/workflows/legacy-v2-api-integration-tests.yml +++ b/.github/workflows/legacy-v2-api-integration-tests.yml @@ -87,11 +87,6 @@ jobs: run: kubectl -n kubeflow port-forward svc/metadata-grpc-service 8080:8080 & continue-on-error: true - - name: Debug TLS cert secret before tests - if: ${{ always() && matrix.pod_to_pod_tls_enabled }} - run: | - kubectl get secret argo-workflows-agent-ca-certificates -n kubeflow - - name: API integration tests v2 id: tests if: ${{ steps.forward-mlmd-port.outcome == 'success' }} @@ -105,7 +100,6 @@ jobs: - name: Collect failed logs if: ${{ steps.deploy.outcome != 'success' || steps.forward-mlmd-port.outcome != 'success' || steps.tests.outcome != 'success' }} run: | - kubectl get secret argo-workflows-agent-ca-certificates -n kubeflow ./.github/resources/scripts/collect-logs.sh --ns kubeflow --output /tmp/tmp_pod_log.txt exit 1 diff --git a/backend/src/driver/api/request.go b/backend/src/driver/api/request.go index e2e73d5cb0a..60b6557792b 100644 --- a/backend/src/driver/api/request.go +++ b/backend/src/driver/api/request.go @@ -39,6 +39,8 @@ type DriverPluginArgs struct { IterationCountPath string `json:"iteration_count_path"` ConditionPath string `json:"condition_path"` PodSpecPathPath string `json:"pod_spec_patch_path"` + MLMDServerAddress string `json:"mlmd_server_address"` + MLMDServerPort string `json:"mlmd_server_port"` MlPipelineServerAddress string `json:"ml_pipeline_server_address"` MlPipelineServerPort string `json:"ml_pipeline_server_port"` MlPipelineTLSEnabled bool `json:"ml_pipeline_tls_enabled"` diff --git a/backend/src/driver/main.go b/backend/src/driver/main.go index 4ebe6c21766..70590b2ea62 100644 --- a/backend/src/driver/main.go +++ b/backend/src/driver/main.go @@ -44,19 +44,8 @@ const ( ) var ( - logLevel = flag.String("log_level", "1", "The verbosity level to log.") - - // config - mlmdServerAddress = flag.String("mlmd_server_address", "", "MLMD server address") - mlmdServerPort = flag.String("mlmd_server_port", "", "MLMD server port") - + logLevel = flag.String("log_level", "1", "The verbosity level to log.") serverPort = flag.String("server_port", ":8080", "Server port") - - // CACertPath Path to the CA certificate - // Default "/etc/ssl/certs/ca-certificates/" is the path where the Argo executor sidecar expects the CA certificate. - // If a Secret named `argo-workflows-agent-ca-certificates` exists in the cluster, - // Argo will automatically mount it here in the sidecar container. - CACertPath = flag.String("ca_cert_path", "/etc/ssl/certs/ca-certificates/", "Path to the CA certificate") ) func main() { @@ -177,5 +166,6 @@ func writeFile(path string, data []byte) (err error) { } func newMlmdClient(mlmdServerAddress string, mlmdServerPort string, tlsCfg *tls.Config) (*metadata.Client, error) { + glog.Infof("mlmd server address: %s:%s\n", mlmdServerAddress, mlmdServerPort) return metadata.NewClient(mlmdServerAddress, mlmdServerPort, tlsCfg) } diff --git a/backend/src/driver/rpc_handler.go b/backend/src/driver/rpc_handler.go index 08649e0db56..0ec398bc957 100644 --- a/backend/src/driver/rpc_handler.go +++ b/backend/src/driver/rpc_handler.go @@ -193,7 +193,7 @@ func drive(args api.DriverPluginArgs) (execution *driver.Execution, err error) { return nil, fmt.Errorf("unable to drive driver: failed to load TLS configuration: %v", err) } } - client, err := newMlmdClient(*mlmdServerAddress, *mlmdServerPort, tlsCfg) + client, err := newMlmdClient(args.MLMDServerAddress, args.MLMDServerPort, tlsCfg) if err != nil { return nil, err } @@ -228,8 +228,8 @@ func drive(args api.DriverPluginArgs) (execution *driver.Execution, err error) { MLPipelineServerAddress: args.MlPipelineServerAddress, MLPipelineServerPort: args.MlPipelineServerPort, MLPipelineTLSEnabled: args.MlPipelineTLSEnabled, - MLMDServerAddress: *mlmdServerAddress, - MLMDServerPort: *mlmdServerPort, + MLMDServerAddress: args.MLMDServerAddress, + MLMDServerPort: args.MLMDServerPort, CaCertPath: args.CACertPath, } diff --git a/backend/src/v2/compiler/argocompiler/argo.go b/backend/src/v2/compiler/argocompiler/argo.go index 321f301e9db..6437c6655b2 100644 --- a/backend/src/v2/compiler/argocompiler/argo.go +++ b/backend/src/v2/compiler/argocompiler/argo.go @@ -136,6 +136,7 @@ func Compile(jobArg *pipelinespec.PipelineJob, kubernetesSpecArg *pipelinespec.S PodMetadata: &wfapi.Metadata{ Annotations: map[string]string{ "pipelines.kubeflow.org/v2_component": "true", + util.AnnotationKeyIstioSidecarInject: util.AnnotationValueIstioSidecarInjectDisabled, }, Labels: map[string]string{ "pipelines.kubeflow.org/v2_component": "true", diff --git a/backend/src/v2/compiler/argocompiler/container.go b/backend/src/v2/compiler/argocompiler/container.go index 2b6513d34f9..beab7413e21 100644 --- a/backend/src/v2/compiler/argocompiler/container.go +++ b/backend/src/v2/compiler/argocompiler/container.go @@ -22,6 +22,7 @@ import ( "strings" "github.com/kubeflow/pipelines/backend/src/v2/config" + "github.com/kubeflow/pipelines/backend/src/v2/metadata" "google.golang.org/protobuf/encoding/protojson" "github.com/kubeflow/pipelines/backend/src/apiserver/config/proxy" @@ -38,9 +39,7 @@ import ( const ( volumeNameKFPLauncher = "kfp-launcher" - volumeNameCABundle = "ca-bundle" LauncherImageEnvVar = "V2_LAUNCHER_IMAGE" - DriverImageEnvVar = "V2_DRIVER_IMAGE" // DefaultLauncherImage & DefaultDriverImage are set as latest here // but are overridden by environment variables set via k8s manifests. // For releases, the manifest will have the correct release version set. @@ -48,9 +47,6 @@ const ( DefaultLauncherImage = "ghcr.io/kubeflow/kfp-launcher:latest" LauncherCommandEnvVar = "V2_LAUNCHER_COMMAND" DefaultLauncherCommand = "launcher-v2" - DefaultDriverImage = "ghcr.io/kubeflow/kfp-driver:latest" - DefaultDriverCommand = "driver" - DriverCommandEnvVar = "V2_DRIVER_COMMAND" PipelineRunAsUserEnvVar = "PIPELINE_RUN_AS_USER" PipelineLogLevelEnvVar = "PIPELINE_LOG_LEVEL" PublishLogsEnvVar = "PUBLISH_LOGS" @@ -199,8 +195,8 @@ func (c *workflowCompiler) addContainerDriverTemplate() (string, error) { "no_proxy": proxy.GetConfig().GetNoProxy(), "ml_pipeline_server_address": config.GetMLPipelineServerConfig().Address, "ml_pipeline_server_port": config.GetMLPipelineServerConfig().Port, - "mlmd_server_address": config.GetMLPipelineServerConfig().Address, - "mlmd_server_port": config.GetMLPipelineServerConfig().Port, + "mlmd_server_address": metadata.GetMetadataConfig().Address, + "mlmd_server_port": metadata.GetMetadataConfig().Port, "cache_disabled": c.cacheDisabled, "ml_pipeline_tls_enabled": c.mlPipelineTLSEnabled, "metadata_tls_enabled": common.GetMetadataTLSEnabled(), diff --git a/backend/src/v2/compiler/argocompiler/dag.go b/backend/src/v2/compiler/argocompiler/dag.go index d03e2c1e900..f8c708516c1 100644 --- a/backend/src/v2/compiler/argocompiler/dag.go +++ b/backend/src/v2/compiler/argocompiler/dag.go @@ -25,6 +25,7 @@ import ( "github.com/kubeflow/pipelines/backend/src/apiserver/config/proxy" "github.com/kubeflow/pipelines/backend/src/v2/compiler" "github.com/kubeflow/pipelines/backend/src/v2/config" + "github.com/kubeflow/pipelines/backend/src/v2/metadata" "k8s.io/apimachinery/pkg/util/intstr" ) @@ -581,8 +582,8 @@ func (c *workflowCompiler) addDAGDriverTemplate() (string, error) { "no_proxy": proxy.GetConfig().GetNoProxy(), "ml_pipeline_server_address": config.GetMLPipelineServerConfig().Address, "ml_pipeline_server_port": config.GetMLPipelineServerConfig().Port, - "mlmd_server_address": config.GetMLPipelineServerConfig().Address, - "mlmd_server_port": config.GetMLPipelineServerConfig().Port, + "mlmd_server_address": metadata.GetMetadataConfig().Address, + "mlmd_server_port": metadata.GetMetadataConfig().Port, "cache_disabled": c.cacheDisabled, "ml_pipeline_tls_enabled": c.mlPipelineTLSEnabled, "metadata_tls_enabled": common.GetMetadataTLSEnabled(), diff --git a/backend/test/end2end/utils/e2e_utils.go b/backend/test/end2end/utils/e2e_utils.go index 0862a005517..32c9983142e 100644 --- a/backend/test/end2end/utils/e2e_utils.go +++ b/backend/test/end2end/utils/e2e_utils.go @@ -76,9 +76,6 @@ func ValidateComponentStatuses(runClient *apiserver.RunClient, k8Client *kuberne expectedTaskDetails := GetTasksFromWorkflow(compiledWorkflow) if *updatedRun.State == run_model.V2beta1RuntimeStateRUNNING { logger.Log("Pipeline run did not finish") - logger.Log("Checking Argo WF events") - events := testutil.DescribeArgoWorkflow(k8Client, *config.Namespace) - ginkgo.AddReportEntry("Argo WF Events", events) logger.Log("Checking workflow controller logs") podLog := testutil.ReadContainerLogs(k8Client, *config.Namespace, "workflow-controller", nil, &testContext.TestStartTimeUTC, config.PodLogLimit) logger.Log("Attaching Workflow Controller logs to the report") @@ -88,9 +85,6 @@ func ValidateComponentStatuses(runClient *apiserver.RunClient, k8Client *kuberne } else { if *updatedRun.State != run_model.V2beta1RuntimeStateSUCCEEDED { logger.Log("Looks like the run %s FAILED, so capture pod logs for the failed task", runID) - logger.Log("Checking Argo WF events") - events := testutil.DescribeArgoWorkflow(k8Client, *config.Namespace) - ginkgo.AddReportEntry("Argo WF Events", events) logger.Log("Checking workflow controller logs") podLog := testutil.ReadContainerLogs(k8Client, *config.Namespace, "workflow-controller", nil, &testContext.TestStartTimeUTC, config.PodLogLimit) logger.Log("Attaching Workflow Controller logs to the report") diff --git a/backend/test/testutil/kubernetes_utils.go b/backend/test/testutil/kubernetes_utils.go index f1ba4ba62b2..9e1acbebf0c 100644 --- a/backend/test/testutil/kubernetes_utils.go +++ b/backend/test/testutil/kubernetes_utils.go @@ -45,22 +45,6 @@ func CreateK8sClient() (*kubernetes.Clientset, error) { return k8sClient, nil } -func DescribeArgoWorkflow(client *kubernetes.Clientset, namespace string) string { - var allEvents []string - events, err := client.CoreV1().Events(namespace).List(context.TODO(), metav1.ListOptions{ - FieldSelector: "involvedObject.kind=Workflow", - }) - if err != nil { - logger.Log("failed to list Argo WF events: %v", err) - } - for _, e := range events.Items { - eventStr := fmt.Sprintf("- name: %s type: %s reason: %s message: %s lastTimestamp: %s", - e.Name, e.Type, e.Reason, e.Message, e.LastTimestamp) - allEvents = append(allEvents, eventStr) - } - return strings.Join(allEvents, ";\n") -} - // ReadContainerLogs - Read pod logs from a specific container func ReadContainerLogs(client *kubernetes.Clientset, namespace string, containerName string, follow *bool, sinceTime *time.Time, logLimit *int64) string { pod := GetPodContainingContainer(client, namespace, containerName) diff --git a/test_data/compiled-workflows/add_numbers.yaml b/test_data/compiled-workflows/add_numbers.yaml index fc0b147d446..902aebbb5b4 100644 --- a/test_data/compiled-workflows/add_numbers.yaml +++ b/test_data/compiled-workflows/add_numbers.yaml @@ -25,8 +25,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -74,8 +79,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: add-numbers pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -85,6 +90,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -121,6 +130,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -150,6 +166,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -160,6 +184,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -253,8 +280,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: add-numbers run_display_name: "" @@ -264,6 +291,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/arguments_parameters.yaml b/test_data/compiled-workflows/arguments_parameters.yaml index 749efba9187..c6824d29660 100644 --- a/test_data/compiled-workflows/arguments_parameters.yaml +++ b/test_data/compiled-workflows/arguments_parameters.yaml @@ -16,8 +16,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -65,8 +70,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: echo pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -76,6 +81,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -112,6 +121,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -141,6 +157,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -151,6 +175,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -244,8 +271,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: echo run_display_name: "" @@ -255,6 +282,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/artifact_cache.yaml b/test_data/compiled-workflows/artifact_cache.yaml index d439fc6c38d..978df7dc867 100644 --- a/test_data/compiled-workflows/artifact_cache.yaml +++ b/test_data/compiled-workflows/artifact_cache.yaml @@ -44,8 +44,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -93,8 +98,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: artifact-cache-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -104,6 +109,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -140,6 +149,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -169,6 +185,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -179,6 +203,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -272,8 +299,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: artifact-cache-pipeline run_display_name: "" @@ -283,6 +310,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/artifact_crust.yaml b/test_data/compiled-workflows/artifact_crust.yaml index a0c5cc7c794..154f875b892 100644 --- a/test_data/compiled-workflows/artifact_crust.yaml +++ b/test_data/compiled-workflows/artifact_crust.yaml @@ -44,8 +44,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -93,8 +98,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: artifact-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -104,6 +109,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -140,6 +149,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -169,6 +185,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -179,6 +203,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -272,8 +299,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: artifact-pipeline run_display_name: "" @@ -283,6 +310,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/artifacts_complex.yaml b/test_data/compiled-workflows/artifacts_complex.yaml index e75f349c59d..b351d31949b 100644 --- a/test_data/compiled-workflows/artifacts_complex.yaml +++ b/test_data/compiled-workflows/artifacts_complex.yaml @@ -70,8 +70,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -119,8 +124,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: math-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -130,6 +135,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -166,6 +175,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -195,6 +211,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -205,6 +229,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -379,8 +406,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: math-pipeline run_display_name: "" @@ -390,6 +417,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/artifacts_simple.yaml b/test_data/compiled-workflows/artifacts_simple.yaml index 6684e132194..19f5f72bb70 100644 --- a/test_data/compiled-workflows/artifacts_simple.yaml +++ b/test_data/compiled-workflows/artifacts_simple.yaml @@ -55,8 +55,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -104,8 +109,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: math-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -115,6 +120,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -151,6 +160,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -180,6 +196,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -190,6 +214,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -283,8 +310,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: math-pipeline run_display_name: "" @@ -294,6 +321,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/collected_artifacts.yaml b/test_data/compiled-workflows/collected_artifacts.yaml index b876e2fe5b5..285fe4ebcfa 100644 --- a/test_data/compiled-workflows/collected_artifacts.yaml +++ b/test_data/compiled-workflows/collected_artifacts.yaml @@ -136,8 +136,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -185,8 +190,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: collected-artifact-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -196,6 +201,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -232,6 +241,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -261,6 +277,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -271,6 +295,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -364,8 +391,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: collected-artifact-pipeline run_display_name: "" @@ -375,6 +402,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/collected_parameters.yaml b/test_data/compiled-workflows/collected_parameters.yaml index 28f87d480d6..cca2e227ddd 100644 --- a/test_data/compiled-workflows/collected_parameters.yaml +++ b/test_data/compiled-workflows/collected_parameters.yaml @@ -71,8 +71,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -120,8 +125,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: collected-param-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -131,6 +136,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -167,6 +176,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -196,6 +212,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -206,6 +230,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -324,8 +351,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: collected-param-pipeline run_display_name: "" @@ -335,6 +362,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/component_with_metadata_fields.yaml b/test_data/compiled-workflows/component_with_metadata_fields.yaml index 2f8ed160cda..ed9f2fe9fd1 100644 --- a/test_data/compiled-workflows/component_with_metadata_fields.yaml +++ b/test_data/compiled-workflows/component_with_metadata_fields.yaml @@ -40,8 +40,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -89,8 +94,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: dataset-joiner pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -100,6 +105,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -136,6 +145,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -165,6 +181,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -175,6 +199,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -268,8 +295,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: dataset-joiner run_display_name: "" @@ -279,6 +306,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/component_with_optional_inputs.yaml b/test_data/compiled-workflows/component_with_optional_inputs.yaml index a9094726b48..11b2f1612f6 100644 --- a/test_data/compiled-workflows/component_with_optional_inputs.yaml +++ b/test_data/compiled-workflows/component_with_optional_inputs.yaml @@ -29,8 +29,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -78,8 +83,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: v2-component-optional-input pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -89,6 +94,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -125,6 +134,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -154,6 +170,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -164,6 +188,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -257,8 +284,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: v2-component-optional-input run_display_name: "" @@ -268,6 +295,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/component_with_pip_index_urls.yaml b/test_data/compiled-workflows/component_with_pip_index_urls.yaml index bcd176cb5b0..30f3f6f4e0b 100644 --- a/test_data/compiled-workflows/component_with_pip_index_urls.yaml +++ b/test_data/compiled-workflows/component_with_pip_index_urls.yaml @@ -27,8 +27,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -76,8 +81,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: v2-component-pip-index-urls pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -87,6 +92,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -123,6 +132,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -152,6 +168,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -162,6 +186,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -255,8 +282,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: v2-component-pip-index-urls run_display_name: "" @@ -266,6 +293,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/component_with_pip_install.yaml b/test_data/compiled-workflows/component_with_pip_install.yaml index 4432b4be9f3..835f6584fa9 100644 --- a/test_data/compiled-workflows/component_with_pip_install.yaml +++ b/test_data/compiled-workflows/component_with_pip_install.yaml @@ -27,8 +27,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -76,8 +81,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: component-with-pip-install pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -87,6 +92,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -123,6 +132,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -152,6 +168,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -162,6 +186,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -255,8 +282,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: component-with-pip-install run_display_name: "" @@ -266,6 +293,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/component_with_pip_install_in_venv.yaml b/test_data/compiled-workflows/component_with_pip_install_in_venv.yaml index 41c9a3d5b2c..72be129ed8a 100644 --- a/test_data/compiled-workflows/component_with_pip_install_in_venv.yaml +++ b/test_data/compiled-workflows/component_with_pip_install_in_venv.yaml @@ -28,8 +28,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -77,8 +82,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: component-with-pip-install pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -88,6 +93,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -124,6 +133,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -153,6 +169,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -163,6 +187,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -256,8 +283,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: component-with-pip-install run_display_name: "" @@ -267,6 +294,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/components_with_optional_artifacts.yaml b/test_data/compiled-workflows/components_with_optional_artifacts.yaml index 72124af42a3..d6806ca1e6b 100644 --- a/test_data/compiled-workflows/components_with_optional_artifacts.yaml +++ b/test_data/compiled-workflows/components_with_optional_artifacts.yaml @@ -41,8 +41,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -90,8 +95,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: optional-artifact-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -101,6 +106,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -137,6 +146,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -166,6 +182,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -176,6 +200,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -303,6 +330,14 @@ spec: requests: cpu: 100m memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault inputs: parameters: - name: task @@ -312,6 +347,10 @@ spec: metadata: {} name: system-importer outputs: {} + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - inputs: parameters: - name: component @@ -358,8 +397,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: optional-artifact-pipeline run_display_name: "" @@ -369,6 +408,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/concat_message.yaml b/test_data/compiled-workflows/concat_message.yaml index 8d2585a6b65..5e6a42c0da5 100644 --- a/test_data/compiled-workflows/concat_message.yaml +++ b/test_data/compiled-workflows/concat_message.yaml @@ -26,8 +26,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -75,8 +80,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: concat-message pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -86,6 +91,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -122,6 +131,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -151,6 +167,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -161,6 +185,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -254,8 +281,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: concat-message run_display_name: "" @@ -265,6 +292,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/conditional_producer_and_consumers.yaml b/test_data/compiled-workflows/conditional_producer_and_consumers.yaml index 1a6e2b4b2ef..a8a9e21815f 100644 --- a/test_data/compiled-workflows/conditional_producer_and_consumers.yaml +++ b/test_data/compiled-workflows/conditional_producer_and_consumers.yaml @@ -47,8 +47,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -96,8 +101,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: math-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -107,6 +112,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -143,6 +152,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -172,6 +188,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -182,6 +206,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -307,8 +334,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: math-pipeline run_display_name: "" @@ -318,6 +345,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/container_component_with_no_inputs.yaml b/test_data/compiled-workflows/container_component_with_no_inputs.yaml index ebc35475dc5..09baa30b63e 100644 --- a/test_data/compiled-workflows/container_component_with_no_inputs.yaml +++ b/test_data/compiled-workflows/container_component_with_no_inputs.yaml @@ -16,8 +16,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -65,8 +70,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: v2-container-component-no-input pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -76,6 +81,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -112,6 +121,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -141,6 +157,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -151,6 +175,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -244,8 +271,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: v2-container-component-no-input run_display_name: "" @@ -255,6 +282,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/container_io.yaml b/test_data/compiled-workflows/container_io.yaml index e9f9ced178b..2ae423682c4 100644 --- a/test_data/compiled-workflows/container_io.yaml +++ b/test_data/compiled-workflows/container_io.yaml @@ -16,8 +16,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -65,8 +70,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: container-io pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -76,6 +81,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -112,6 +121,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -141,6 +157,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -151,6 +175,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -244,8 +271,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: container-io run_display_name: "" @@ -255,6 +282,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/container_no_input.yaml b/test_data/compiled-workflows/container_no_input.yaml index 5bbb4e40e48..041ba8af4ba 100644 --- a/test_data/compiled-workflows/container_no_input.yaml +++ b/test_data/compiled-workflows/container_no_input.yaml @@ -16,8 +16,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -65,8 +70,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: container-no-input pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -76,6 +81,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -112,6 +121,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -141,6 +157,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -151,6 +175,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -244,8 +271,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: container-no-input run_display_name: "" @@ -255,6 +282,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/container_with_artifact_output.yaml b/test_data/compiled-workflows/container_with_artifact_output.yaml index e948722513f..0e521562f9e 100644 --- a/test_data/compiled-workflows/container_with_artifact_output.yaml +++ b/test_data/compiled-workflows/container_with_artifact_output.yaml @@ -16,8 +16,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -65,8 +70,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: container-with-artifact-output pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -76,6 +81,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -112,6 +121,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -141,6 +157,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -151,6 +175,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -244,8 +271,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: container-with-artifact-output run_display_name: "" @@ -255,6 +282,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/container_with_concat_placeholder.yaml b/test_data/compiled-workflows/container_with_concat_placeholder.yaml index cb91b0f16cc..663235d2510 100644 --- a/test_data/compiled-workflows/container_with_concat_placeholder.yaml +++ b/test_data/compiled-workflows/container_with_concat_placeholder.yaml @@ -17,8 +17,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -66,8 +71,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: container-with-concat-placeholder pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -77,6 +82,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -113,6 +122,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -142,6 +158,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -152,6 +176,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -245,8 +272,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: container-with-concat-placeholder run_display_name: "" @@ -256,6 +283,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/container_with_if_placeholder.yaml b/test_data/compiled-workflows/container_with_if_placeholder.yaml index 0c48844c104..c09fdf9fee6 100644 --- a/test_data/compiled-workflows/container_with_if_placeholder.yaml +++ b/test_data/compiled-workflows/container_with_if_placeholder.yaml @@ -19,8 +19,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -68,8 +73,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: container-with-if-placeholder pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -79,6 +84,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -115,6 +124,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -144,6 +160,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -154,6 +178,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -247,8 +274,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: container-with-if-placeholder run_display_name: "" @@ -258,6 +285,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/container_with_placeholder_in_fstring.yaml b/test_data/compiled-workflows/container_with_placeholder_in_fstring.yaml index 4ff8d8f59be..05e0341960a 100644 --- a/test_data/compiled-workflows/container_with_placeholder_in_fstring.yaml +++ b/test_data/compiled-workflows/container_with_placeholder_in_fstring.yaml @@ -16,8 +16,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -65,8 +70,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: container-with-placeholder-in-fstring pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -76,6 +81,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -112,6 +121,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -141,6 +157,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -151,6 +175,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -244,8 +271,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: container-with-placeholder-in-fstring run_display_name: "" @@ -255,6 +282,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/containerized_python_component.yaml b/test_data/compiled-workflows/containerized_python_component.yaml index 7ca099e2273..16b9841507c 100644 --- a/test_data/compiled-workflows/containerized_python_component.yaml +++ b/test_data/compiled-workflows/containerized_python_component.yaml @@ -16,8 +16,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -65,8 +70,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: concat-message pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -76,6 +81,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -112,6 +121,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -141,6 +157,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -151,6 +175,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -244,8 +271,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: concat-message run_display_name: "" @@ -255,6 +282,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/create_pod_metadata_complex.yaml b/test_data/compiled-workflows/create_pod_metadata_complex.yaml index c6f0e2274ef..2fc4424f582 100644 --- a/test_data/compiled-workflows/create_pod_metadata_complex.yaml +++ b/test_data/compiled-workflows/create_pod_metadata_complex.yaml @@ -69,8 +69,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -118,8 +123,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-pod-metadata pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -129,6 +134,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -165,6 +174,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -194,6 +210,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -204,6 +228,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -273,6 +300,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -302,6 +336,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -323,6 +365,9 @@ spec: name: metadata-1-2-system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -386,6 +431,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -415,6 +467,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -432,6 +492,9 @@ spec: name: metadata-2-0-system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -597,8 +660,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-pod-metadata run_display_name: "" @@ -608,6 +671,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/cross_loop_after_topology.yaml b/test_data/compiled-workflows/cross_loop_after_topology.yaml index 1b2f9a16df2..cf827c9acb5 100644 --- a/test_data/compiled-workflows/cross_loop_after_topology.yaml +++ b/test_data/compiled-workflows/cross_loop_after_topology.yaml @@ -46,8 +46,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -95,8 +100,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: my-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -106,6 +111,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -142,6 +151,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -171,6 +187,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -181,6 +205,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -306,8 +333,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: my-pipeline run_display_name: "" @@ -317,6 +344,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/dict_input.yaml b/test_data/compiled-workflows/dict_input.yaml index 99376dec431..a5ecd6c66bf 100644 --- a/test_data/compiled-workflows/dict_input.yaml +++ b/test_data/compiled-workflows/dict_input.yaml @@ -25,8 +25,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -74,8 +79,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: dict-input pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -85,6 +90,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -121,6 +130,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -150,6 +166,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -160,6 +184,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -253,8 +280,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: dict-input run_display_name: "" @@ -264,6 +291,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/embedded_artifact.yaml b/test_data/compiled-workflows/embedded_artifact.yaml index 5cd04e9bf68..5af008eee78 100644 --- a/test_data/compiled-workflows/embedded_artifact.yaml +++ b/test_data/compiled-workflows/embedded_artifact.yaml @@ -66,8 +66,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -115,8 +120,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: nb-simple pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -126,6 +131,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -162,6 +171,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -191,6 +207,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -201,6 +225,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -318,8 +345,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: nb-simple run_display_name: "" @@ -329,6 +356,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/env-var.yaml b/test_data/compiled-workflows/env-var.yaml index ad16e797f76..c49587b759f 100644 --- a/test_data/compiled-workflows/env-var.yaml +++ b/test_data/compiled-workflows/env-var.yaml @@ -27,8 +27,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -76,8 +81,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: test-env-exists pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -87,6 +92,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -123,6 +132,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -152,6 +168,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -162,6 +186,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -255,8 +282,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: test-env-exists run_display_name: "" @@ -266,6 +293,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/fail_v2.yaml b/test_data/compiled-workflows/fail_v2.yaml index 278898c0fb0..dbf44a3cc51 100644 --- a/test_data/compiled-workflows/fail_v2.yaml +++ b/test_data/compiled-workflows/fail_v2.yaml @@ -25,8 +25,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -74,8 +79,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: fail-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -85,6 +90,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -121,6 +130,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -150,6 +166,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -160,6 +184,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -253,8 +280,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: fail-pipeline run_display_name: "" @@ -264,6 +291,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/flip_coin.yaml b/test_data/compiled-workflows/flip_coin.yaml index 59c2ea3964a..8c8006ad189 100644 --- a/test_data/compiled-workflows/flip_coin.yaml +++ b/test_data/compiled-workflows/flip_coin.yaml @@ -82,8 +82,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -131,8 +136,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: conditional-execution-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -142,6 +147,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -178,6 +187,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -207,6 +223,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -217,6 +241,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -346,8 +373,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: conditional-execution-pipeline run_display_name: "" @@ -357,6 +384,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/hello_world.yaml b/test_data/compiled-workflows/hello_world.yaml index 696efcb8d40..43a7466b391 100644 --- a/test_data/compiled-workflows/hello_world.yaml +++ b/test_data/compiled-workflows/hello_world.yaml @@ -16,8 +16,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -65,8 +70,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: echo pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -76,6 +81,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -112,6 +121,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -141,6 +157,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -151,6 +175,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -244,8 +271,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: echo run_display_name: "" @@ -255,6 +282,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/identity.yaml b/test_data/compiled-workflows/identity.yaml index 93e8fe36409..f0a3f534295 100644 --- a/test_data/compiled-workflows/identity.yaml +++ b/test_data/compiled-workflows/identity.yaml @@ -25,8 +25,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -74,8 +79,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: identity pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -85,6 +90,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -121,6 +130,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -150,6 +166,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -160,6 +184,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -253,8 +280,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: identity run_display_name: "" @@ -264,6 +291,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/if_elif_else_complex.yaml b/test_data/compiled-workflows/if_elif_else_complex.yaml index 05b4a3301de..5f10f993fe7 100644 --- a/test_data/compiled-workflows/if_elif_else_complex.yaml +++ b/test_data/compiled-workflows/if_elif_else_complex.yaml @@ -116,8 +116,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -165,8 +170,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: lucky-number-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -176,6 +181,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -212,6 +221,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -241,6 +257,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -251,6 +275,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -345,8 +372,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: lucky-number-pipeline run_display_name: "" @@ -356,6 +383,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/if_elif_else_with_oneof_parameters.yaml b/test_data/compiled-workflows/if_elif_else_with_oneof_parameters.yaml index cc5d02e1fe3..9cd5108d621 100644 --- a/test_data/compiled-workflows/if_elif_else_with_oneof_parameters.yaml +++ b/test_data/compiled-workflows/if_elif_else_with_oneof_parameters.yaml @@ -71,8 +71,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -120,8 +125,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: outer-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -131,6 +136,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -167,6 +176,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -196,6 +212,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -206,6 +230,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -365,8 +392,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: outer-pipeline run_display_name: "" @@ -376,6 +403,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/if_else_with_oneof_artifacts.yaml b/test_data/compiled-workflows/if_else_with_oneof_artifacts.yaml index 1116b8d2a6e..ad1af9414fd 100644 --- a/test_data/compiled-workflows/if_else_with_oneof_artifacts.yaml +++ b/test_data/compiled-workflows/if_else_with_oneof_artifacts.yaml @@ -63,8 +63,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -112,8 +117,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: outer-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -123,6 +128,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -159,6 +168,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -188,6 +204,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -198,6 +222,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -323,8 +350,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: outer-pipeline run_display_name: "" @@ -334,6 +361,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/if_else_with_oneof_parameters.yaml b/test_data/compiled-workflows/if_else_with_oneof_parameters.yaml index a1670b09c48..d516e13ce59 100644 --- a/test_data/compiled-workflows/if_else_with_oneof_parameters.yaml +++ b/test_data/compiled-workflows/if_else_with_oneof_parameters.yaml @@ -49,8 +49,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -98,8 +103,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: flip-coin-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -109,6 +114,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -145,6 +154,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -174,6 +190,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -184,6 +208,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -311,8 +338,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: flip-coin-pipeline run_display_name: "" @@ -322,6 +349,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/input_artifact.yaml b/test_data/compiled-workflows/input_artifact.yaml index 54354700ae5..f88410099a6 100644 --- a/test_data/compiled-workflows/input_artifact.yaml +++ b/test_data/compiled-workflows/input_artifact.yaml @@ -25,8 +25,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -74,8 +79,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: input-artifact pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -85,6 +90,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -121,6 +130,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -150,6 +166,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -160,6 +184,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -253,8 +280,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: input-artifact run_display_name: "" @@ -264,6 +291,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/iris_pipeline_compiled.yaml b/test_data/compiled-workflows/iris_pipeline_compiled.yaml index 479f4f15b5e..ee1eb93c9be 100644 --- a/test_data/compiled-workflows/iris_pipeline_compiled.yaml +++ b/test_data/compiled-workflows/iris_pipeline_compiled.yaml @@ -78,8 +78,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -127,8 +132,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: iris-training-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -138,6 +143,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -174,6 +183,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -203,6 +219,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -213,6 +237,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -356,8 +383,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: iris-training-pipeline run_display_name: "" @@ -367,6 +394,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/lightweight_python_functions_pipeline.yaml b/test_data/compiled-workflows/lightweight_python_functions_pipeline.yaml index 64a68aa9208..dbfbb57c065 100644 --- a/test_data/compiled-workflows/lightweight_python_functions_pipeline.yaml +++ b/test_data/compiled-workflows/lightweight_python_functions_pipeline.yaml @@ -80,8 +80,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -129,8 +134,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: my-test-pipeline-beta pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -140,6 +145,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -176,6 +185,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -205,6 +221,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -215,6 +239,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -333,8 +360,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: my-test-pipeline-beta run_display_name: "" @@ -344,6 +371,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/lightweight_python_functions_with_outputs.yaml b/test_data/compiled-workflows/lightweight_python_functions_with_outputs.yaml index 3398b654e3f..02fc701cce2 100644 --- a/test_data/compiled-workflows/lightweight_python_functions_with_outputs.yaml +++ b/test_data/compiled-workflows/lightweight_python_functions_with_outputs.yaml @@ -72,8 +72,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -121,8 +126,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: functions-with-outputs pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -132,6 +137,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -168,6 +177,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -197,6 +213,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -207,6 +231,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -374,8 +401,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: functions-with-outputs run_display_name: "" @@ -385,6 +412,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/log_streaming_compiled.yaml b/test_data/compiled-workflows/log_streaming_compiled.yaml index 666408a637c..75417b87c56 100644 --- a/test_data/compiled-workflows/log_streaming_compiled.yaml +++ b/test_data/compiled-workflows/log_streaming_compiled.yaml @@ -29,8 +29,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -78,8 +83,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: log-streaming-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -89,6 +94,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -125,6 +134,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -154,6 +170,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -164,6 +188,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -257,8 +284,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: log-streaming-pipeline run_display_name: "" @@ -268,6 +295,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/long-running.yaml b/test_data/compiled-workflows/long-running.yaml index 893ba8194ac..436e40fc2bd 100644 --- a/test_data/compiled-workflows/long-running.yaml +++ b/test_data/compiled-workflows/long-running.yaml @@ -16,8 +16,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -65,8 +70,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: wait-awhile pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -76,6 +81,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -112,6 +121,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -141,6 +157,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -151,6 +175,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -269,8 +296,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: wait-awhile run_display_name: "" @@ -280,6 +307,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/loop_consume_upstream.yaml b/test_data/compiled-workflows/loop_consume_upstream.yaml index 188878488bf..99e18f624e5 100644 --- a/test_data/compiled-workflows/loop_consume_upstream.yaml +++ b/test_data/compiled-workflows/loop_consume_upstream.yaml @@ -73,8 +73,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -122,8 +127,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: loop-consume-upstream pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -133,6 +138,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -169,6 +178,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -198,6 +214,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -208,6 +232,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -328,8 +355,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: loop-consume-upstream run_display_name: "" @@ -339,6 +366,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/metrics_visualization_v2.yaml b/test_data/compiled-workflows/metrics_visualization_v2.yaml index 0488064f133..a32e2ce8cb6 100644 --- a/test_data/compiled-workflows/metrics_visualization_v2.yaml +++ b/test_data/compiled-workflows/metrics_visualization_v2.yaml @@ -117,8 +117,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -166,8 +171,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: metrics-visualization-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -177,6 +182,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -213,6 +222,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -242,6 +258,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -252,6 +276,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -441,8 +468,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: metrics-visualization-pipeline run_display_name: "" @@ -452,6 +479,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/missing_kubernetes_optional_inputs.yaml b/test_data/compiled-workflows/missing_kubernetes_optional_inputs.yaml index 7b959e66c29..22e96b994db 100644 --- a/test_data/compiled-workflows/missing_kubernetes_optional_inputs.yaml +++ b/test_data/compiled-workflows/missing_kubernetes_optional_inputs.yaml @@ -28,8 +28,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -77,8 +82,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: missing-kubernetes-optional-inputs-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -88,6 +93,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -124,6 +133,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -153,6 +169,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -163,6 +187,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -259,8 +286,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: missing-kubernetes-optional-inputs-pipeline run_display_name: "" @@ -270,6 +297,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/mixed_parameters.yaml b/test_data/compiled-workflows/mixed_parameters.yaml index 85de6b4d77d..150d04236a8 100644 --- a/test_data/compiled-workflows/mixed_parameters.yaml +++ b/test_data/compiled-workflows/mixed_parameters.yaml @@ -42,8 +42,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -91,8 +96,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: mixed-parameters-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -102,6 +107,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -138,6 +147,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -167,6 +183,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -177,6 +201,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -270,8 +297,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: mixed-parameters-pipeline run_display_name: "" @@ -281,6 +308,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/modelcar.yaml b/test_data/compiled-workflows/modelcar.yaml index ec222cff1c8..a7d0df40797 100644 --- a/test_data/compiled-workflows/modelcar.yaml +++ b/test_data/compiled-workflows/modelcar.yaml @@ -50,8 +50,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -99,8 +104,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-modelcar-model pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -110,6 +115,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -146,6 +155,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -175,6 +191,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -185,6 +209,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -248,6 +275,14 @@ spec: requests: cpu: 100m memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault inputs: parameters: - name: task @@ -257,6 +292,10 @@ spec: metadata: {} name: system-importer outputs: {} + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -372,8 +411,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-modelcar-model run_display_name: "" @@ -383,6 +422,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/mounted_cabundle_configmap.yaml b/test_data/compiled-workflows/mounted_cabundle_configmap.yaml index d3e8b33945d..354b50e2b1e 100644 --- a/test_data/compiled-workflows/mounted_cabundle_configmap.yaml +++ b/test_data/compiled-workflows/mounted_cabundle_configmap.yaml @@ -16,8 +16,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -66,8 +71,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: echo pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -77,6 +82,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -113,6 +122,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -144,6 +160,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -154,6 +178,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -254,8 +281,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: echo run_display_name: "" @@ -265,6 +292,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/mounted_cabundle_secret.yaml b/test_data/compiled-workflows/mounted_cabundle_secret.yaml index ecc0c6853ad..660c63fb0d9 100644 --- a/test_data/compiled-workflows/mounted_cabundle_secret.yaml +++ b/test_data/compiled-workflows/mounted_cabundle_secret.yaml @@ -16,8 +16,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -66,8 +71,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: echo pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -77,6 +82,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -113,6 +122,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -144,6 +160,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -154,6 +178,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -254,8 +281,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: echo run_display_name: "" @@ -265,6 +292,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/multiple_artifacts_namedtuple.yaml b/test_data/compiled-workflows/multiple_artifacts_namedtuple.yaml index dead7d3dabd..df967cb979e 100644 --- a/test_data/compiled-workflows/multiple_artifacts_namedtuple.yaml +++ b/test_data/compiled-workflows/multiple_artifacts_namedtuple.yaml @@ -46,8 +46,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -95,8 +100,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: multiple-artifacts-namedtuple-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -106,6 +111,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -142,6 +151,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -171,6 +187,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -181,6 +205,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -274,8 +301,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: multiple-artifacts-namedtuple-pipeline run_display_name: "" @@ -285,6 +312,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/multiple_parameters_namedtuple.yaml b/test_data/compiled-workflows/multiple_parameters_namedtuple.yaml index 974347920b0..c0600885313 100644 --- a/test_data/compiled-workflows/multiple_parameters_namedtuple.yaml +++ b/test_data/compiled-workflows/multiple_parameters_namedtuple.yaml @@ -45,8 +45,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -94,8 +99,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: multiple-parameters-namedtuple-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -105,6 +110,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -141,6 +150,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -170,6 +186,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -180,6 +204,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -273,8 +300,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: multiple-parameters-namedtuple-pipeline run_display_name: "" @@ -284,6 +311,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/nested_pipeline_opt_input_child_level_compiled.yaml b/test_data/compiled-workflows/nested_pipeline_opt_input_child_level_compiled.yaml index 0a79b693e14..253f3bfe06e 100644 --- a/test_data/compiled-workflows/nested_pipeline_opt_input_child_level_compiled.yaml +++ b/test_data/compiled-workflows/nested_pipeline_opt_input_child_level_compiled.yaml @@ -105,8 +105,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -154,8 +159,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: nested-pipeline-opt-input-child-level pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -165,6 +170,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -201,6 +210,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -230,6 +246,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -240,6 +264,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -453,8 +480,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: nested-pipeline-opt-input-child-level run_display_name: "" @@ -464,6 +491,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml b/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml index 29e4e1949d6..e87ab36b379 100644 --- a/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml +++ b/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml @@ -56,8 +56,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -105,8 +110,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: nested-pipeline-opt-inputs-nil pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -116,6 +121,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -152,6 +161,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -181,6 +197,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -191,6 +215,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -332,8 +359,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: nested-pipeline-opt-inputs-nil run_display_name: "" @@ -343,6 +370,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/nested_pipeline_opt_inputs_parent_level_compiled.yaml b/test_data/compiled-workflows/nested_pipeline_opt_inputs_parent_level_compiled.yaml index 41163c0f4f7..d621e217aa5 100644 --- a/test_data/compiled-workflows/nested_pipeline_opt_inputs_parent_level_compiled.yaml +++ b/test_data/compiled-workflows/nested_pipeline_opt_inputs_parent_level_compiled.yaml @@ -108,8 +108,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -157,8 +162,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: nested-pipeline-opt-inputs-parent-level pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -168,6 +173,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -204,6 +213,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -233,6 +249,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -243,6 +267,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -464,8 +491,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: nested-pipeline-opt-inputs-parent-level run_display_name: "" @@ -475,6 +502,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/nested_return.yaml b/test_data/compiled-workflows/nested_return.yaml index 3c7c4ded5ab..84f3e3b4621 100644 --- a/test_data/compiled-workflows/nested_return.yaml +++ b/test_data/compiled-workflows/nested_return.yaml @@ -26,8 +26,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -75,8 +80,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: nested-return pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -86,6 +91,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -122,6 +131,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -151,6 +167,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -161,6 +185,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -254,8 +281,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: nested-return run_display_name: "" @@ -265,6 +292,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/nested_with_parameters.yaml b/test_data/compiled-workflows/nested_with_parameters.yaml index 25e6ec23585..0b1345376c6 100644 --- a/test_data/compiled-workflows/nested_with_parameters.yaml +++ b/test_data/compiled-workflows/nested_with_parameters.yaml @@ -58,8 +58,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -107,8 +112,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: math-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -118,6 +123,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -154,6 +163,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -183,6 +199,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -193,6 +217,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -335,8 +362,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: math-pipeline run_display_name: "" @@ -346,6 +373,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/notebook_component_mixed.yaml b/test_data/compiled-workflows/notebook_component_mixed.yaml index b474cdff968..3fcd9ced595 100644 --- a/test_data/compiled-workflows/notebook_component_mixed.yaml +++ b/test_data/compiled-workflows/notebook_component_mixed.yaml @@ -254,8 +254,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -303,8 +308,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: nb-mixed pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -314,6 +319,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -350,6 +359,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -379,6 +395,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -389,6 +413,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -532,8 +559,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: nb-mixed run_display_name: "" @@ -543,6 +570,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/notebook_component_simple.yaml b/test_data/compiled-workflows/notebook_component_simple.yaml index c6aaaf82fa8..2cced3b2af7 100644 --- a/test_data/compiled-workflows/notebook_component_simple.yaml +++ b/test_data/compiled-workflows/notebook_component_simple.yaml @@ -123,8 +123,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -172,8 +177,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: nb-simple pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -183,6 +188,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -219,6 +228,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -248,6 +264,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -258,6 +282,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -351,8 +378,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: nb-simple run_display_name: "" @@ -362,6 +389,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/output_metrics.yaml b/test_data/compiled-workflows/output_metrics.yaml index f11d3fe9041..63fea53f231 100644 --- a/test_data/compiled-workflows/output_metrics.yaml +++ b/test_data/compiled-workflows/output_metrics.yaml @@ -27,8 +27,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -76,8 +81,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: output-metrics pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -87,6 +92,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -123,6 +132,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -152,6 +168,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -162,6 +186,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -255,8 +282,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: output-metrics run_display_name: "" @@ -266,6 +293,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/parallel_for_after_dependency.yaml b/test_data/compiled-workflows/parallel_for_after_dependency.yaml index 5660c5da9a0..61f9bc377a2 100644 --- a/test_data/compiled-workflows/parallel_for_after_dependency.yaml +++ b/test_data/compiled-workflows/parallel_for_after_dependency.yaml @@ -28,8 +28,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -77,8 +82,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: loop-with-after-dependency-set pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -88,6 +93,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -124,6 +133,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -153,6 +169,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -163,6 +187,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -256,8 +283,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: loop-with-after-dependency-set run_display_name: "" @@ -267,6 +294,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/parameter_cache.yaml b/test_data/compiled-workflows/parameter_cache.yaml index c1d0509dd24..64adae7491b 100644 --- a/test_data/compiled-workflows/parameter_cache.yaml +++ b/test_data/compiled-workflows/parameter_cache.yaml @@ -42,8 +42,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -91,8 +96,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: parameter-cache-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -102,6 +107,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -138,6 +147,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -167,6 +183,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -177,6 +201,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -270,8 +297,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: parameter-cache-pipeline run_display_name: "" @@ -281,6 +308,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/parameter_oneof.yaml b/test_data/compiled-workflows/parameter_oneof.yaml index dedfe7d7604..161abdf2991 100644 --- a/test_data/compiled-workflows/parameter_oneof.yaml +++ b/test_data/compiled-workflows/parameter_oneof.yaml @@ -81,8 +81,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -130,8 +135,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: parameter-oneof-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -141,6 +146,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -177,6 +186,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -206,6 +222,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -216,6 +240,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -343,8 +370,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: parameter-oneof-pipeline run_display_name: "" @@ -354,6 +381,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/parameters_complex.yaml b/test_data/compiled-workflows/parameters_complex.yaml index c70d767894e..0b768cf534a 100644 --- a/test_data/compiled-workflows/parameters_complex.yaml +++ b/test_data/compiled-workflows/parameters_complex.yaml @@ -75,8 +75,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -124,8 +129,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: math-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -135,6 +140,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -171,6 +180,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -200,6 +216,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -210,6 +234,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -303,8 +330,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: math-pipeline run_display_name: "" @@ -314,6 +341,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/parameters_simple.yaml b/test_data/compiled-workflows/parameters_simple.yaml index 67acd9b47c7..08cc9c1766c 100644 --- a/test_data/compiled-workflows/parameters_simple.yaml +++ b/test_data/compiled-workflows/parameters_simple.yaml @@ -46,8 +46,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -95,8 +100,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: math-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -106,6 +111,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -142,6 +151,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -171,6 +187,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -181,6 +205,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -274,8 +301,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: math-pipeline run_display_name: "" @@ -285,6 +312,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_as_exit_task.yaml b/test_data/compiled-workflows/pipeline_as_exit_task.yaml index fc08e3ce6f8..e8920879925 100644 --- a/test_data/compiled-workflows/pipeline_as_exit_task.yaml +++ b/test_data/compiled-workflows/pipeline_as_exit_task.yaml @@ -62,8 +62,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -111,8 +116,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-task-final-status-conditional pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -122,6 +127,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -158,6 +167,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -187,6 +203,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -197,6 +221,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -291,8 +318,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-task-final-status-conditional run_display_name: "" @@ -302,6 +329,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_in_pipeline.yaml b/test_data/compiled-workflows/pipeline_in_pipeline.yaml index 903b2a1aacb..f2ef62dca8b 100644 --- a/test_data/compiled-workflows/pipeline_in_pipeline.yaml +++ b/test_data/compiled-workflows/pipeline_in_pipeline.yaml @@ -31,8 +31,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -80,8 +85,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-in-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -91,6 +96,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -127,6 +136,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -156,6 +172,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -166,6 +190,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -284,8 +311,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-in-pipeline run_display_name: "" @@ -295,6 +322,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml b/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml index a09d11030ac..cd56cf69db9 100644 --- a/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml +++ b/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml @@ -40,8 +40,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -89,8 +94,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-in-pipeline-complex pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -100,6 +105,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -136,6 +145,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -165,6 +181,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -175,6 +199,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -300,8 +327,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-in-pipeline-complex run_display_name: "" @@ -311,6 +338,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_in_pipeline_loaded_from_yaml.yaml b/test_data/compiled-workflows/pipeline_in_pipeline_loaded_from_yaml.yaml index 171f5e0eccd..e0c5f98f183 100644 --- a/test_data/compiled-workflows/pipeline_in_pipeline_loaded_from_yaml.yaml +++ b/test_data/compiled-workflows/pipeline_in_pipeline_loaded_from_yaml.yaml @@ -47,8 +47,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -96,8 +101,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-in-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -107,6 +112,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -143,6 +152,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -172,6 +188,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -182,6 +206,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -300,8 +327,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-in-pipeline run_display_name: "" @@ -311,6 +338,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_producer_consumer.yaml b/test_data/compiled-workflows/pipeline_producer_consumer.yaml index 471414e2931..46e7800a487 100644 --- a/test_data/compiled-workflows/pipeline_producer_consumer.yaml +++ b/test_data/compiled-workflows/pipeline_producer_consumer.yaml @@ -78,8 +78,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -127,8 +132,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: math-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -138,6 +143,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -174,6 +183,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -203,6 +219,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -213,6 +237,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -306,8 +333,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: math-pipeline run_display_name: "" @@ -317,6 +344,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_after.yaml b/test_data/compiled-workflows/pipeline_with_after.yaml index c2ba28efb62..dc7acde896f 100644 --- a/test_data/compiled-workflows/pipeline_with_after.yaml +++ b/test_data/compiled-workflows/pipeline_with_after.yaml @@ -19,8 +19,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -68,8 +73,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-after pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -79,6 +84,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -115,6 +124,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -144,6 +160,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -154,6 +178,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -300,8 +327,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-after run_display_name: "" @@ -311,6 +338,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_artifact_custom_path.yaml b/test_data/compiled-workflows/pipeline_with_artifact_custom_path.yaml index cdface75e1f..f34f2413c0e 100644 --- a/test_data/compiled-workflows/pipeline_with_artifact_custom_path.yaml +++ b/test_data/compiled-workflows/pipeline_with_artifact_custom_path.yaml @@ -47,8 +47,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -96,8 +101,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-custom-path-artifact pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -107,6 +112,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -143,6 +152,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -172,6 +188,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -182,6 +206,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -300,8 +327,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-custom-path-artifact run_display_name: "" @@ -311,6 +338,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_artifact_upload_download.yaml b/test_data/compiled-workflows/pipeline_with_artifact_upload_download.yaml index 54c3a3419b0..5bf452566ce 100644 --- a/test_data/compiled-workflows/pipeline_with_artifact_upload_download.yaml +++ b/test_data/compiled-workflows/pipeline_with_artifact_upload_download.yaml @@ -46,8 +46,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -95,8 +100,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-datasets pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -106,6 +111,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -142,6 +151,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -171,6 +187,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -181,6 +205,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -299,8 +326,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-datasets run_display_name: "" @@ -310,6 +337,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_concat_placeholder.yaml b/test_data/compiled-workflows/pipeline_with_concat_placeholder.yaml index 9719d0b1753..9337c16c7ce 100644 --- a/test_data/compiled-workflows/pipeline_with_concat_placeholder.yaml +++ b/test_data/compiled-workflows/pipeline_with_concat_placeholder.yaml @@ -18,8 +18,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -67,8 +72,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: one-step-pipeline-with-concat-placeholder pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -78,6 +83,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -114,6 +123,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -143,6 +159,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -153,6 +177,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -246,8 +273,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: one-step-pipeline-with-concat-placeholder run_display_name: "" @@ -257,6 +284,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_condition.yaml b/test_data/compiled-workflows/pipeline_with_condition.yaml index c25ada2c18c..8b4ee02814c 100644 --- a/test_data/compiled-workflows/pipeline_with_condition.yaml +++ b/test_data/compiled-workflows/pipeline_with_condition.yaml @@ -44,8 +44,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -93,8 +98,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: single-condition-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -104,6 +109,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -140,6 +149,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -169,6 +185,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -179,6 +203,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -321,8 +348,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: single-condition-pipeline run_display_name: "" @@ -332,6 +359,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml b/test_data/compiled-workflows/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml index 23ee85a61f4..63a5de13675 100644 --- a/test_data/compiled-workflows/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml +++ b/test_data/compiled-workflows/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml @@ -141,8 +141,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -190,8 +195,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-dynamic-condition-output pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -201,6 +206,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -237,6 +246,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -266,6 +282,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -276,6 +300,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -369,8 +396,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-dynamic-condition-output run_display_name: "" @@ -380,6 +407,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_dynamic_importer_metadata.yaml b/test_data/compiled-workflows/pipeline_with_dynamic_importer_metadata.yaml index 1d65d0e6a1e..64e4298f07f 100644 --- a/test_data/compiled-workflows/pipeline_with_dynamic_importer_metadata.yaml +++ b/test_data/compiled-workflows/pipeline_with_dynamic_importer_metadata.yaml @@ -33,8 +33,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - container: @@ -85,6 +90,14 @@ spec: requests: cpu: 100m memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault inputs: parameters: - name: task @@ -94,6 +107,10 @@ spec: metadata: {} name: system-importer outputs: {} + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - inputs: parameters: - name: component @@ -139,8 +156,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-importer pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -150,6 +167,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -186,6 +207,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -215,6 +243,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -225,6 +261,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -342,8 +381,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-importer run_display_name: "" @@ -353,6 +392,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_dynamic_task_output_custom_training_job.yaml b/test_data/compiled-workflows/pipeline_with_dynamic_task_output_custom_training_job.yaml index 8cffe38bb92..5a77fdbc063 100644 --- a/test_data/compiled-workflows/pipeline_with_dynamic_task_output_custom_training_job.yaml +++ b/test_data/compiled-workflows/pipeline_with_dynamic_task_output_custom_training_job.yaml @@ -103,8 +103,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -152,8 +157,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -163,6 +168,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -199,6 +208,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -228,6 +244,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -238,6 +262,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -404,8 +431,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline run_display_name: "" @@ -415,6 +442,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_env.yaml b/test_data/compiled-workflows/pipeline_with_env.yaml index 56c6a52fa63..dd841a15348 100644 --- a/test_data/compiled-workflows/pipeline_with_env.yaml +++ b/test_data/compiled-workflows/pipeline_with_env.yaml @@ -31,8 +31,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -80,8 +85,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-env pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -91,6 +96,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -127,6 +136,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -156,6 +172,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -166,6 +190,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -283,8 +310,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-env run_display_name: "" @@ -294,6 +321,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_exit_handler.yaml b/test_data/compiled-workflows/pipeline_with_exit_handler.yaml index b775ede6d65..604ae68924b 100644 --- a/test_data/compiled-workflows/pipeline_with_exit_handler.yaml +++ b/test_data/compiled-workflows/pipeline_with_exit_handler.yaml @@ -43,8 +43,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -92,8 +97,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-exit-handler pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -103,6 +108,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -139,6 +148,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -168,6 +184,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -178,6 +202,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -329,8 +356,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-exit-handler run_display_name: "" @@ -340,6 +367,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_google_artifact_type.yaml b/test_data/compiled-workflows/pipeline_with_google_artifact_type.yaml index 02a3e082d24..0be0ff47892 100644 --- a/test_data/compiled-workflows/pipeline_with_google_artifact_type.yaml +++ b/test_data/compiled-workflows/pipeline_with_google_artifact_type.yaml @@ -51,8 +51,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - container: @@ -103,6 +108,14 @@ spec: requests: cpu: 100m memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault inputs: parameters: - name: task @@ -112,6 +125,10 @@ spec: metadata: {} name: system-importer outputs: {} + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - inputs: parameters: - name: component @@ -157,8 +174,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-google-types pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -168,6 +185,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -204,6 +225,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -233,6 +261,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -243,6 +279,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -373,8 +412,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-google-types run_display_name: "" @@ -384,6 +423,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_importer.yaml b/test_data/compiled-workflows/pipeline_with_importer.yaml index 4b8b899fe59..3af33e3be15 100644 --- a/test_data/compiled-workflows/pipeline_with_importer.yaml +++ b/test_data/compiled-workflows/pipeline_with_importer.yaml @@ -41,8 +41,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - container: @@ -93,6 +98,14 @@ spec: requests: cpu: 100m memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault inputs: parameters: - name: task @@ -102,6 +115,10 @@ spec: metadata: {} name: system-importer outputs: {} + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - inputs: parameters: - name: component @@ -147,8 +164,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-importer pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -158,6 +175,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -194,6 +215,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -223,6 +251,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -233,6 +269,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -339,8 +378,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-importer run_display_name: "" @@ -350,6 +389,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_importer_and_gcpc_types.yaml b/test_data/compiled-workflows/pipeline_with_importer_and_gcpc_types.yaml index 89723e6c36f..a9af7bb8798 100644 --- a/test_data/compiled-workflows/pipeline_with_importer_and_gcpc_types.yaml +++ b/test_data/compiled-workflows/pipeline_with_importer_and_gcpc_types.yaml @@ -20,8 +20,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -69,8 +74,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-importer-and-gcpc-type pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -80,6 +85,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -116,6 +125,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -145,6 +161,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -155,6 +179,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -218,6 +245,14 @@ spec: requests: cpu: 100m memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault inputs: parameters: - name: task @@ -227,6 +262,10 @@ spec: metadata: {} name: system-importer outputs: {} + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -318,8 +357,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-importer-and-gcpc-type run_display_name: "" @@ -329,6 +368,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_importer_workspace.yaml b/test_data/compiled-workflows/pipeline_with_importer_workspace.yaml index ba35cd8d45c..29ac513e0ba 100644 --- a/test_data/compiled-workflows/pipeline_with_importer_workspace.yaml +++ b/test_data/compiled-workflows/pipeline_with_importer_workspace.yaml @@ -108,8 +108,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - container: @@ -160,6 +165,14 @@ spec: requests: cpu: 100m memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-workspace name: kfp-workspace @@ -172,6 +185,10 @@ spec: metadata: {} name: system-importer-workspace outputs: {} + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumes: - name: kfp-workspace persistentVolumeClaim: @@ -221,8 +238,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-importer-workspace pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -232,6 +249,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -268,6 +289,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -297,6 +325,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -307,6 +343,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -450,8 +489,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-importer-workspace run_display_name: "" @@ -461,6 +500,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_input_status_state.yaml b/test_data/compiled-workflows/pipeline_with_input_status_state.yaml index 4c45fd09f1d..fde4b1105de 100644 --- a/test_data/compiled-workflows/pipeline_with_input_status_state.yaml +++ b/test_data/compiled-workflows/pipeline_with_input_status_state.yaml @@ -43,8 +43,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -92,8 +97,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: status-state-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -103,6 +108,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -139,6 +148,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -168,6 +184,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -178,6 +202,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -303,8 +330,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: status-state-pipeline run_display_name: "" @@ -314,6 +341,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_loops.yaml b/test_data/compiled-workflows/pipeline_with_loops.yaml index b0c280c1c23..f31b91a63de 100644 --- a/test_data/compiled-workflows/pipeline_with_loops.yaml +++ b/test_data/compiled-workflows/pipeline_with_loops.yaml @@ -59,8 +59,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -108,8 +113,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-loops pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -119,6 +124,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -155,6 +164,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -184,6 +200,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -194,6 +218,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -447,8 +474,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-loops run_display_name: "" @@ -458,6 +485,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_loops_and_conditions.yaml b/test_data/compiled-workflows/pipeline_with_loops_and_conditions.yaml index 61c5053bc03..1fe33c8ba53 100644 --- a/test_data/compiled-workflows/pipeline_with_loops_and_conditions.yaml +++ b/test_data/compiled-workflows/pipeline_with_loops_and_conditions.yaml @@ -105,8 +105,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -154,8 +159,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-loops-and-conditions-multi-layers pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -165,6 +170,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -201,6 +210,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -230,6 +246,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -240,6 +264,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -397,8 +424,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-loops-and-conditions-multi-layers run_display_name: "" @@ -408,6 +435,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_metadata_fields.yaml b/test_data/compiled-workflows/pipeline_with_metadata_fields.yaml index bbb5888b6c3..421f17ee670 100644 --- a/test_data/compiled-workflows/pipeline_with_metadata_fields.yaml +++ b/test_data/compiled-workflows/pipeline_with_metadata_fields.yaml @@ -56,8 +56,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -105,8 +110,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: dataset-concatenator pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -116,6 +121,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -152,6 +161,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -181,6 +197,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -191,6 +215,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -309,8 +336,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: dataset-concatenator run_display_name: "" @@ -320,6 +347,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_metrics_outputs.yaml b/test_data/compiled-workflows/pipeline_with_metrics_outputs.yaml index 8fd715ef07f..69adb3797f8 100644 --- a/test_data/compiled-workflows/pipeline_with_metrics_outputs.yaml +++ b/test_data/compiled-workflows/pipeline_with_metrics_outputs.yaml @@ -30,8 +30,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -79,8 +84,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-metrics-outputs pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -90,6 +95,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -126,6 +135,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -155,6 +171,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -165,6 +189,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -258,8 +285,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-metrics-outputs run_display_name: "" @@ -269,6 +296,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_multiple_exit_handlers.yaml b/test_data/compiled-workflows/pipeline_with_multiple_exit_handlers.yaml index 9b668c211a6..8b0a9f97198 100644 --- a/test_data/compiled-workflows/pipeline_with_multiple_exit_handlers.yaml +++ b/test_data/compiled-workflows/pipeline_with_multiple_exit_handlers.yaml @@ -49,8 +49,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -98,8 +103,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-multiple-exit-handlers pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -109,6 +114,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -145,6 +154,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -174,6 +190,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -184,6 +208,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -465,8 +492,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-multiple-exit-handlers run_display_name: "" @@ -476,6 +503,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_nested_conditions.yaml b/test_data/compiled-workflows/pipeline_with_nested_conditions.yaml index e06399dbf85..7d9cc3db54c 100644 --- a/test_data/compiled-workflows/pipeline_with_nested_conditions.yaml +++ b/test_data/compiled-workflows/pipeline_with_nested_conditions.yaml @@ -46,8 +46,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -95,8 +100,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: nested-conditions-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -106,6 +111,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -142,6 +151,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -171,6 +187,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -181,6 +205,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -299,8 +326,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: nested-conditions-pipeline run_display_name: "" @@ -310,6 +337,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_nested_conditions_yaml.yaml b/test_data/compiled-workflows/pipeline_with_nested_conditions_yaml.yaml index 87e8481fb37..1f910249bc2 100644 --- a/test_data/compiled-workflows/pipeline_with_nested_conditions_yaml.yaml +++ b/test_data/compiled-workflows/pipeline_with_nested_conditions_yaml.yaml @@ -58,8 +58,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -107,8 +112,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: conditional-execution-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -118,6 +123,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -154,6 +163,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -183,6 +199,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -193,6 +217,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -322,8 +349,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: conditional-execution-pipeline run_display_name: "" @@ -333,6 +360,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_nested_loops.yaml b/test_data/compiled-workflows/pipeline_with_nested_loops.yaml index 47973a344f8..dbc04c58adb 100644 --- a/test_data/compiled-workflows/pipeline_with_nested_loops.yaml +++ b/test_data/compiled-workflows/pipeline_with_nested_loops.yaml @@ -36,8 +36,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -85,8 +90,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-nested-loops pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -96,6 +101,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -132,6 +141,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -161,6 +177,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -171,6 +195,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -264,8 +291,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-nested-loops run_display_name: "" @@ -275,6 +302,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_only_display_name.yaml b/test_data/compiled-workflows/pipeline_with_only_display_name.yaml index a448e2e9282..add6d913166 100644 --- a/test_data/compiled-workflows/pipeline_with_only_display_name.yaml +++ b/test_data/compiled-workflows/pipeline_with_only_display_name.yaml @@ -16,8 +16,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -65,8 +70,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: echo-name pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -76,6 +81,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -112,6 +121,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -141,6 +157,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -151,6 +175,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -244,8 +271,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: echo-name run_display_name: "" @@ -255,6 +282,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_outputs.yaml b/test_data/compiled-workflows/pipeline_with_outputs.yaml index 546f09b4dc4..832e2226059 100644 --- a/test_data/compiled-workflows/pipeline_with_outputs.yaml +++ b/test_data/compiled-workflows/pipeline_with_outputs.yaml @@ -32,8 +32,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -81,8 +86,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-in-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -92,6 +97,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -128,6 +137,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -157,6 +173,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -167,6 +191,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -285,8 +312,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-in-pipeline run_display_name: "" @@ -296,6 +323,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_parallelfor_parallelism.yaml b/test_data/compiled-workflows/pipeline_with_parallelfor_parallelism.yaml index 512de22662f..65d909d2f56 100644 --- a/test_data/compiled-workflows/pipeline_with_parallelfor_parallelism.yaml +++ b/test_data/compiled-workflows/pipeline_with_parallelfor_parallelism.yaml @@ -123,8 +123,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -172,8 +177,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-loops pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -183,6 +188,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -219,6 +228,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -248,6 +264,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -258,6 +282,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -351,8 +378,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-loops run_display_name: "" @@ -362,6 +389,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_params_containing_format.yaml b/test_data/compiled-workflows/pipeline_with_params_containing_format.yaml index 6a3d3bcda05..eb0e3b2db7c 100644 --- a/test_data/compiled-workflows/pipeline_with_params_containing_format.yaml +++ b/test_data/compiled-workflows/pipeline_with_params_containing_format.yaml @@ -45,8 +45,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -94,8 +99,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-pipelineparam-containing-format pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -105,6 +110,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -141,6 +150,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -170,6 +186,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -180,6 +204,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -274,8 +301,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-pipelineparam-containing-format run_display_name: "" @@ -285,6 +312,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_placeholders.yaml b/test_data/compiled-workflows/pipeline_with_placeholders.yaml index d6bb902be12..d4b020f95ca 100644 --- a/test_data/compiled-workflows/pipeline_with_placeholders.yaml +++ b/test_data/compiled-workflows/pipeline_with_placeholders.yaml @@ -32,8 +32,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -81,8 +86,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-placeholders pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -92,6 +97,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -128,6 +137,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -157,6 +173,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -167,6 +191,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -260,8 +287,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-placeholders run_display_name: "" @@ -271,6 +298,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_pod_metadata.yaml b/test_data/compiled-workflows/pipeline_with_pod_metadata.yaml index 902083337d2..0006d6536cc 100644 --- a/test_data/compiled-workflows/pipeline_with_pod_metadata.yaml +++ b/test_data/compiled-workflows/pipeline_with_pod_metadata.yaml @@ -147,8 +147,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -196,8 +201,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-pod-metadata pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -207,6 +212,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -243,6 +252,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -272,6 +288,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -282,6 +306,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -351,6 +378,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -380,6 +414,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -401,6 +443,9 @@ spec: name: metadata-1-2-system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -494,6 +539,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -523,6 +575,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -556,6 +616,9 @@ spec: name: metadata-4-3-system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -619,6 +682,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -648,6 +718,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -665,6 +743,9 @@ spec: name: metadata-2-0-system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -734,6 +815,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -763,6 +851,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -783,6 +879,9 @@ spec: name: metadata-0-3-system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -1042,8 +1141,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-pod-metadata run_display_name: "" @@ -1053,6 +1152,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_retry.yaml b/test_data/compiled-workflows/pipeline_with_retry.yaml index dc5f6f295b9..7fb9adc9c1d 100644 --- a/test_data/compiled-workflows/pipeline_with_retry.yaml +++ b/test_data/compiled-workflows/pipeline_with_retry.yaml @@ -25,8 +25,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -74,8 +79,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: test-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -85,6 +90,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -137,6 +146,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -166,6 +182,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -186,6 +210,9 @@ spec: factor: '{{inputs.parameters.retry-backoff-factor}}' maxDuration: '{{inputs.parameters.retry-backoff-max-duration}}' limit: '{{inputs.parameters.retry-max-count}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -287,8 +314,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: test-pipeline run_display_name: "" @@ -298,6 +325,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_reused_component.yaml b/test_data/compiled-workflows/pipeline_with_reused_component.yaml index 45969af5ab6..c077325071e 100644 --- a/test_data/compiled-workflows/pipeline_with_reused_component.yaml +++ b/test_data/compiled-workflows/pipeline_with_reused_component.yaml @@ -25,8 +25,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -74,8 +79,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-reused-component pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -85,6 +90,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -121,6 +130,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -150,6 +166,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -160,6 +184,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -303,8 +330,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-reused-component run_display_name: "" @@ -314,6 +341,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml b/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml index 60e484458d7..de483de2be4 100644 --- a/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml +++ b/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml @@ -44,8 +44,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -93,8 +98,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-secret-env pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -104,6 +109,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -140,6 +149,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -169,6 +185,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -179,6 +203,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -299,8 +326,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-secret-env run_display_name: "" @@ -310,6 +337,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_secret_as_volume.yaml b/test_data/compiled-workflows/pipeline_with_secret_as_volume.yaml index 2b65aa802f2..b1ba7fe8409 100644 --- a/test_data/compiled-workflows/pipeline_with_secret_as_volume.yaml +++ b/test_data/compiled-workflows/pipeline_with_secret_as_volume.yaml @@ -32,8 +32,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -81,8 +86,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-secret-volume pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -92,6 +97,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -128,6 +137,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -157,6 +173,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -167,6 +191,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -262,8 +289,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-secret-volume run_display_name: "" @@ -273,6 +300,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_string_machine_fields_pipeline_input.yaml b/test_data/compiled-workflows/pipeline_with_string_machine_fields_pipeline_input.yaml index 5b98555e739..b6a53429642 100644 --- a/test_data/compiled-workflows/pipeline_with_string_machine_fields_pipeline_input.yaml +++ b/test_data/compiled-workflows/pipeline_with_string_machine_fields_pipeline_input.yaml @@ -25,8 +25,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -74,8 +79,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -85,6 +90,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -121,6 +130,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -150,6 +166,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -160,6 +184,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -253,8 +280,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline run_display_name: "" @@ -264,6 +291,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml b/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml index 68e0552496f..54e9b21f420 100644 --- a/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml +++ b/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml @@ -77,8 +77,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -126,8 +131,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -137,6 +142,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -173,6 +182,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -202,6 +218,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -212,6 +236,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -403,8 +430,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline run_display_name: "" @@ -414,6 +441,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_submit_request.yaml b/test_data/compiled-workflows/pipeline_with_submit_request.yaml index 6f509a4a96f..c01c33d78ec 100644 --- a/test_data/compiled-workflows/pipeline_with_submit_request.yaml +++ b/test_data/compiled-workflows/pipeline_with_submit_request.yaml @@ -36,8 +36,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -85,8 +90,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-external-request pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -96,6 +101,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -132,6 +141,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -161,6 +177,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -171,6 +195,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -288,8 +315,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-external-request run_display_name: "" @@ -299,6 +326,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_task_final_status.yaml b/test_data/compiled-workflows/pipeline_with_task_final_status.yaml index 0d90f158029..33d57996017 100644 --- a/test_data/compiled-workflows/pipeline_with_task_final_status.yaml +++ b/test_data/compiled-workflows/pipeline_with_task_final_status.yaml @@ -59,8 +59,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -108,8 +113,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-task-final-status pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -119,6 +124,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -155,6 +164,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -184,6 +200,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -194,6 +218,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -344,8 +371,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-task-final-status run_display_name: "" @@ -355,6 +382,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_task_final_status_yaml.yaml b/test_data/compiled-workflows/pipeline_with_task_final_status_yaml.yaml index b494e7015fb..941f1ef2c8f 100644 --- a/test_data/compiled-workflows/pipeline_with_task_final_status_yaml.yaml +++ b/test_data/compiled-workflows/pipeline_with_task_final_status_yaml.yaml @@ -24,8 +24,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -73,8 +78,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-task-final-status-yaml pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -84,6 +89,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -120,6 +129,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -149,6 +165,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -159,6 +183,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -284,8 +311,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-task-final-status-yaml run_display_name: "" @@ -295,6 +322,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_task_using_ignore_upstream_failure.yaml b/test_data/compiled-workflows/pipeline_with_task_using_ignore_upstream_failure.yaml index 3855f39c411..c645ffa99d4 100644 --- a/test_data/compiled-workflows/pipeline_with_task_using_ignore_upstream_failure.yaml +++ b/test_data/compiled-workflows/pipeline_with_task_using_ignore_upstream_failure.yaml @@ -39,8 +39,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -88,8 +93,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: my-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -99,6 +104,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -135,6 +144,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -164,6 +180,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -174,6 +198,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -306,8 +333,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: my-pipeline run_display_name: "" @@ -317,6 +344,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_utils.yaml b/test_data/compiled-workflows/pipeline_with_utils.yaml index edd22889735..b874a5268e1 100644 --- a/test_data/compiled-workflows/pipeline_with_utils.yaml +++ b/test_data/compiled-workflows/pipeline_with_utils.yaml @@ -29,8 +29,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -78,8 +83,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-utils pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -89,6 +94,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -125,6 +134,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -154,6 +170,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -164,6 +188,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -257,8 +284,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-utils run_display_name: "" @@ -268,6 +295,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_various_io_types.yaml b/test_data/compiled-workflows/pipeline_with_various_io_types.yaml index 4644b8fd764..1016d977ca4 100644 --- a/test_data/compiled-workflows/pipeline_with_various_io_types.yaml +++ b/test_data/compiled-workflows/pipeline_with_various_io_types.yaml @@ -20,8 +20,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -69,8 +74,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-various-types pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -80,6 +85,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -116,6 +125,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -145,6 +161,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -155,6 +179,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -273,8 +300,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-various-types run_display_name: "" @@ -284,6 +311,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_volume.yaml b/test_data/compiled-workflows/pipeline_with_volume.yaml index c3a1b47250a..b0bb74e9f0f 100644 --- a/test_data/compiled-workflows/pipeline_with_volume.yaml +++ b/test_data/compiled-workflows/pipeline_with_volume.yaml @@ -73,8 +73,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -122,8 +127,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-volume pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -133,6 +138,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -169,6 +178,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -198,6 +214,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -208,6 +232,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -360,8 +387,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-volume run_display_name: "" @@ -371,6 +398,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_volume_no_cache.yaml b/test_data/compiled-workflows/pipeline_with_volume_no_cache.yaml index ae5f8bcc57f..e3e0f571864 100644 --- a/test_data/compiled-workflows/pipeline_with_volume_no_cache.yaml +++ b/test_data/compiled-workflows/pipeline_with_volume_no_cache.yaml @@ -73,8 +73,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -122,8 +127,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-volume-no-cache pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -133,6 +138,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -169,6 +178,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -198,6 +214,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -208,6 +232,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -360,8 +387,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-volume-no-cache run_display_name: "" @@ -371,6 +398,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_workspace.yaml b/test_data/compiled-workflows/pipeline_with_workspace.yaml index 66e7339481e..6f6cdfe067f 100644 --- a/test_data/compiled-workflows/pipeline_with_workspace.yaml +++ b/test_data/compiled-workflows/pipeline_with_workspace.yaml @@ -49,8 +49,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -98,8 +103,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-workspace pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -109,6 +114,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -145,6 +154,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -174,6 +190,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -184,6 +208,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -302,8 +329,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pipeline-with-workspace run_display_name: "" @@ -313,6 +340,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/placeholder_with_if_placeholder_none_input_value.yaml b/test_data/compiled-workflows/placeholder_with_if_placeholder_none_input_value.yaml index 93a1104219f..3cf07a1a0ed 100644 --- a/test_data/compiled-workflows/placeholder_with_if_placeholder_none_input_value.yaml +++ b/test_data/compiled-workflows/placeholder_with_if_placeholder_none_input_value.yaml @@ -19,8 +19,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -68,8 +73,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: one-step-pipeline-with-if-placeholder-supply-none pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -79,6 +84,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -115,6 +124,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -144,6 +160,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -154,6 +178,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -247,8 +274,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: one-step-pipeline-with-if-placeholder-supply-none run_display_name: "" @@ -258,6 +285,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/preprocess.yaml b/test_data/compiled-workflows/preprocess.yaml index 4e6ea81351c..8501a6d591f 100644 --- a/test_data/compiled-workflows/preprocess.yaml +++ b/test_data/compiled-workflows/preprocess.yaml @@ -47,8 +47,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -96,8 +101,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: preprocess pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -107,6 +112,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -143,6 +152,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -172,6 +188,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -182,6 +206,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -275,8 +302,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: preprocess run_display_name: "" @@ -286,6 +313,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/producer_consumer_param_pipeline.yaml b/test_data/compiled-workflows/producer_consumer_param_pipeline.yaml index c0278e3e534..90ee21a7cc3 100644 --- a/test_data/compiled-workflows/producer_consumer_param_pipeline.yaml +++ b/test_data/compiled-workflows/producer_consumer_param_pipeline.yaml @@ -23,8 +23,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -72,8 +77,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: producer-consumer-param-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -83,6 +88,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -119,6 +128,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -148,6 +164,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -158,6 +182,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -276,8 +303,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: producer-consumer-param-pipeline run_display_name: "" @@ -287,6 +314,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pvc_mount.yaml b/test_data/compiled-workflows/pvc_mount.yaml index 565bd879543..c4f83b53063 100644 --- a/test_data/compiled-workflows/pvc_mount.yaml +++ b/test_data/compiled-workflows/pvc_mount.yaml @@ -43,8 +43,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -92,8 +97,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pvc-mount-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -103,6 +108,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -139,6 +148,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -168,6 +184,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -178,6 +202,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -300,8 +327,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pvc-mount-pipeline run_display_name: "" @@ -311,6 +338,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml b/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml index 35e54a94b6c..6db48d8a78e 100644 --- a/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml +++ b/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml @@ -36,8 +36,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - container: @@ -88,6 +93,14 @@ spec: requests: cpu: 100m memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault inputs: parameters: - name: task @@ -97,6 +110,10 @@ spec: metadata: {} name: system-importer outputs: {} + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - inputs: parameters: - name: component @@ -142,8 +159,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: make-language-model-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -153,6 +170,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -189,6 +210,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -218,6 +246,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -228,6 +264,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -334,8 +373,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: make-language-model-pipeline run_display_name: "" @@ -345,6 +384,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pythonic_artifacts_test_pipeline.yaml b/test_data/compiled-workflows/pythonic_artifacts_test_pipeline.yaml index edaac75d2ae..79c2e43a913 100644 --- a/test_data/compiled-workflows/pythonic_artifacts_test_pipeline.yaml +++ b/test_data/compiled-workflows/pythonic_artifacts_test_pipeline.yaml @@ -44,8 +44,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -93,8 +98,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pythonic-artifacts-test pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -104,6 +109,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -140,6 +149,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -169,6 +185,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -179,6 +203,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -297,8 +324,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: pythonic-artifacts-test run_display_name: "" @@ -308,6 +335,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pythonic_artifacts_with_list_of_artifacts.yaml b/test_data/compiled-workflows/pythonic_artifacts_with_list_of_artifacts.yaml index 8a5b4a5c0d6..dcbbef39511 100644 --- a/test_data/compiled-workflows/pythonic_artifacts_with_list_of_artifacts.yaml +++ b/test_data/compiled-workflows/pythonic_artifacts_with_list_of_artifacts.yaml @@ -45,8 +45,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -94,8 +99,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: make-and-join-datasets pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -105,6 +110,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -141,6 +150,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -170,6 +186,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -180,6 +204,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -273,8 +300,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: make-and-join-datasets run_display_name: "" @@ -284,6 +311,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/pythonic_artifacts_with_multiple_returns.yaml b/test_data/compiled-workflows/pythonic_artifacts_with_multiple_returns.yaml index df32472aab0..032e1fa41e2 100644 --- a/test_data/compiled-workflows/pythonic_artifacts_with_multiple_returns.yaml +++ b/test_data/compiled-workflows/pythonic_artifacts_with_multiple_returns.yaml @@ -50,8 +50,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -99,8 +104,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: split-datasets-and-return-first pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -110,6 +115,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -146,6 +155,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -175,6 +191,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -185,6 +209,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -278,8 +305,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: split-datasets-and-return-first run_display_name: "" @@ -289,6 +316,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/ray_integration_compiled.yaml b/test_data/compiled-workflows/ray_integration_compiled.yaml index a467221790f..a1bf92c1da7 100644 --- a/test_data/compiled-workflows/ray_integration_compiled.yaml +++ b/test_data/compiled-workflows/ray_integration_compiled.yaml @@ -90,8 +90,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -139,8 +144,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: ray-integration-test pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -150,6 +155,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -186,6 +195,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -215,6 +231,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -225,6 +249,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -318,8 +345,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: ray-integration-test run_display_name: "" @@ -329,6 +356,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/run_as_user_cache_disabled.yaml b/test_data/compiled-workflows/run_as_user_cache_disabled.yaml index 49c2f423924..0509104b2ba 100644 --- a/test_data/compiled-workflows/run_as_user_cache_disabled.yaml +++ b/test_data/compiled-workflows/run_as_user_cache_disabled.yaml @@ -16,9 +16,11 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: + runAsNonRoot: true runAsUser: 1001 seccompProfile: type: RuntimeDefault @@ -70,8 +72,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: echo pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -81,6 +83,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -117,6 +123,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -148,6 +161,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -158,6 +179,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -252,8 +276,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: echo run_display_name: "" @@ -263,6 +287,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/run_as_user_cache_enabled.yaml b/test_data/compiled-workflows/run_as_user_cache_enabled.yaml index 49c2f423924..0509104b2ba 100644 --- a/test_data/compiled-workflows/run_as_user_cache_enabled.yaml +++ b/test_data/compiled-workflows/run_as_user_cache_enabled.yaml @@ -16,9 +16,11 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: + runAsNonRoot: true runAsUser: 1001 seccompProfile: type: RuntimeDefault @@ -70,8 +72,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: echo pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -81,6 +83,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -117,6 +123,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -148,6 +161,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -158,6 +179,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -252,8 +276,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: echo run_display_name: "" @@ -263,6 +287,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/sequential_v1.yaml b/test_data/compiled-workflows/sequential_v1.yaml index 116e8a40c49..2c76d1e86b4 100644 --- a/test_data/compiled-workflows/sequential_v1.yaml +++ b/test_data/compiled-workflows/sequential_v1.yaml @@ -16,8 +16,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -65,8 +70,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: sequential pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -76,6 +81,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -112,6 +121,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -141,6 +157,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -151,6 +175,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -268,8 +295,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: sequential run_display_name: "" @@ -279,6 +306,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/sequential_v2.yaml b/test_data/compiled-workflows/sequential_v2.yaml index 1de49f9e028..20e6a0b2cc1 100644 --- a/test_data/compiled-workflows/sequential_v2.yaml +++ b/test_data/compiled-workflows/sequential_v2.yaml @@ -20,8 +20,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -69,8 +74,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: sequential pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -80,6 +85,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -116,6 +125,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -145,6 +161,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -155,6 +179,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -273,8 +300,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: sequential run_display_name: "" @@ -284,6 +311,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/take_nap_compiled.yaml b/test_data/compiled-workflows/take_nap_compiled.yaml index 5152d37a480..2e69c97f814 100644 --- a/test_data/compiled-workflows/take_nap_compiled.yaml +++ b/test_data/compiled-workflows/take_nap_compiled.yaml @@ -41,8 +41,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -90,8 +95,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: take-nap-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -101,6 +106,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -137,6 +146,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -166,6 +182,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -176,6 +200,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -294,8 +321,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: take-nap-pipeline run_display_name: "" @@ -305,6 +332,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/take_nap_pipeline_root_compiled.yaml b/test_data/compiled-workflows/take_nap_pipeline_root_compiled.yaml index 5152d37a480..2e69c97f814 100644 --- a/test_data/compiled-workflows/take_nap_pipeline_root_compiled.yaml +++ b/test_data/compiled-workflows/take_nap_pipeline_root_compiled.yaml @@ -41,8 +41,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -90,8 +95,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: take-nap-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -101,6 +106,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -137,6 +146,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -166,6 +182,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -176,6 +200,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -294,8 +321,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: take-nap-pipeline run_display_name: "" @@ -305,6 +332,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/two_step_pipeline.yaml b/test_data/compiled-workflows/two_step_pipeline.yaml index 6159ad7b8fa..94aef95181d 100644 --- a/test_data/compiled-workflows/two_step_pipeline.yaml +++ b/test_data/compiled-workflows/two_step_pipeline.yaml @@ -21,8 +21,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -70,8 +75,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: simple-two-step-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -81,6 +86,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -117,6 +126,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -146,6 +162,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -156,6 +180,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -274,8 +301,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: simple-two-step-pipeline run_display_name: "" @@ -285,6 +312,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/two_step_pipeline_containerized.yaml b/test_data/compiled-workflows/two_step_pipeline_containerized.yaml index f8cc8cd32e5..9e2b9d852d7 100644 --- a/test_data/compiled-workflows/two_step_pipeline_containerized.yaml +++ b/test_data/compiled-workflows/two_step_pipeline_containerized.yaml @@ -22,8 +22,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -71,8 +76,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: containerized-two-step-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -82,6 +87,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -118,6 +127,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -147,6 +163,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -157,6 +181,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -275,8 +302,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: containerized-two-step-pipeline run_display_name: "" @@ -286,6 +313,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/upload_download_compiled.yaml b/test_data/compiled-workflows/upload_download_compiled.yaml index c155980628a..803f5a3517a 100644 --- a/test_data/compiled-workflows/upload_download_compiled.yaml +++ b/test_data/compiled-workflows/upload_download_compiled.yaml @@ -80,8 +80,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -129,8 +134,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: test-data-passing-pipeline-1 pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -140,6 +145,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -176,6 +185,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -205,6 +221,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -215,6 +239,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -358,8 +385,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: test-data-passing-pipeline-1 run_display_name: "" @@ -369,6 +396,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: diff --git a/test_data/compiled-workflows/xgboost_sample_pipeline.yaml b/test_data/compiled-workflows/xgboost_sample_pipeline.yaml index afd01ca8d8e..302700a6945 100644 --- a/test_data/compiled-workflows/xgboost_sample_pipeline.yaml +++ b/test_data/compiled-workflows/xgboost_sample_pipeline.yaml @@ -284,8 +284,13 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault serviceAccountName: pipeline-runner templates: - inputs: @@ -333,8 +338,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: xgboost-sample-pipeline pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' @@ -344,6 +349,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: CONTAINER + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: @@ -380,6 +389,13 @@ spec: image: gcr.io/ml-pipeline/should-be-overridden-during-runtime name: "" resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -409,6 +425,14 @@ spec: memory: 256Mi requests: cpu: 100m + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /kfp-launcher name: kfp-launcher @@ -419,6 +443,9 @@ spec: name: system-container-impl outputs: {} podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - emptyDir: {} name: kfp-launcher @@ -688,8 +715,8 @@ spec: ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local ml_pipeline_server_port: "8887" ml_pipeline_tls_enabled: false - mlmd_server_address: ml-pipeline.kubeflow.svc.cluster.local - mlmd_server_port: "8887" + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" no_proxy: "" pipeline_name: xgboost-sample-pipeline run_display_name: "" @@ -699,6 +726,10 @@ spec: task: '{{inputs.parameters.task}}' task_name: '{{inputs.parameters.task-name}}' type: '{{inputs.parameters.driver-type}}' + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault - dag: tasks: - arguments: From 33d46e8fd55e0db4a6ee990b6212fdf4c58e072f Mon Sep 17 00:00:00 2001 From: arpechenin Date: Mon, 23 Feb 2026 23:25:19 +0300 Subject: [PATCH 26/51] - make driver-plugin securityContext PodSecurity restricted compliant Signed-off-by: arpechenin --- .../resources/manifests/base/driver-plugin-cm-path.yaml | 6 ++++++ .../standalone/tls-enabled/driver-plugin-cm-path.yaml | 9 ++++++++- .../base/pipeline/ml-pipeline-driver-plugin-cm.yaml | 6 ++++++ .../patches/ml-pipeline-driver-plugin-cm.yaml | 6 ++++++ manifests/kustomize/env/dev/driver-plugin-cm-path.yaml | 8 +++++++- 5 files changed, 33 insertions(+), 2 deletions(-) diff --git a/.github/resources/manifests/base/driver-plugin-cm-path.yaml b/.github/resources/manifests/base/driver-plugin-cm-path.yaml index a2a32ba8c3f..385c997f974 100644 --- a/.github/resources/manifests/base/driver-plugin-cm-path.yaml +++ b/.github/resources/manifests/base/driver-plugin-cm-path.yaml @@ -19,4 +19,10 @@ data: securityContext: runAsNonRoot: true runAsUser: 65534 + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault diff --git a/.github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml b/.github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml index ed43ddf7e6f..91117642a3d 100644 --- a/.github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml +++ b/.github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml @@ -5,7 +5,8 @@ metadata: data: sidecar.container: | name: driver-plugin - image: kind-registry:5000/driver + image: kind-registry:5000/driver:latest + imagePullPolicy: IfNotPresent ports: - containerPort: 8080 resources: @@ -18,6 +19,12 @@ data: securityContext: runAsNonRoot: true runAsUser: 65534 + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - name: argo-workflows-agent-ca-certificates mountPath: /kfp/certs diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml index d34e659d6b4..aca2f9fb22e 100644 --- a/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml +++ b/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml @@ -21,3 +21,9 @@ data: securityContext: runAsNonRoot: true runAsUser: 65534 + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-driver-plugin-cm.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-driver-plugin-cm.yaml index b8dfca5af76..06cc855769d 100644 --- a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-driver-plugin-cm.yaml +++ b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-driver-plugin-cm.yaml @@ -21,6 +21,12 @@ data: securityContext: runAsNonRoot: true runAsUser: 65534 + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - name: argo-workflows-agent-ca-certificates mountPath: /kfp/certs diff --git a/manifests/kustomize/env/dev/driver-plugin-cm-path.yaml b/manifests/kustomize/env/dev/driver-plugin-cm-path.yaml index 15951f419bc..049d2b09a14 100644 --- a/manifests/kustomize/env/dev/driver-plugin-cm-path.yaml +++ b/manifests/kustomize/env/dev/driver-plugin-cm-path.yaml @@ -17,4 +17,10 @@ data: memory: 512Mi securityContext: runAsNonRoot: true - runAsUser: 65534 \ No newline at end of file + runAsUser: 65534 + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault \ No newline at end of file From 64c5bda50cead79c428398f49770e7491fa718a8 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Tue, 24 Feb 2026 00:38:17 +0300 Subject: [PATCH 27/51] - pass mlmdtls parameter to launcher cmd args Signed-off-by: arpechenin --- backend/src/driver/rpc_handler.go | 1 + .../pipeline_with_string_machine_fields_task_output.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/backend/src/driver/rpc_handler.go b/backend/src/driver/rpc_handler.go index 0ec398bc957..fa2e629a3cb 100644 --- a/backend/src/driver/rpc_handler.go +++ b/backend/src/driver/rpc_handler.go @@ -230,6 +230,7 @@ func drive(args api.DriverPluginArgs) (execution *driver.Execution, err error) { MLPipelineTLSEnabled: args.MlPipelineTLSEnabled, MLMDServerAddress: args.MLMDServerAddress, MLMDServerPort: args.MLMDServerPort, + MLMDTLSEnabled: args.MetadataTLSEnabled, CaCertPath: args.CACertPath, } diff --git a/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml b/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml index 54e9b21f420..ffc87b9199c 100644 --- a/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml +++ b/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml @@ -72,7 +72,7 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef sum_numbers(a: int, b: int) -\u003e int:\n return a + b\n\n"],"image":"python:3.11","resources":{"accelerator":{"resourceCount":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}","resourceType":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"},"resourceCpuLimit":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}","resourceMemoryLimit":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}}' - name: components-root - value: '{"dag":{"tasks":{"accelerator-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-limit"},"taskInfo":{"name":"accelerator-limit"}},"accelerator-type":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-type"},"taskInfo":{"name":"accelerator-type"}},"cpu-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-cpu-limit"},"taskInfo":{"name":"cpu-limit"}},"memory-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-memory-limit"},"taskInfo":{"name":"memory-limit"}},"sum-numbers":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-limit","accelerator-type","cpu-limit","memory-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}}}}' + value: '{"dag":{"tasks":{"accelerator-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-limit"},"taskInfo":{"name":"accelerator-limit"}},"accelerator-type":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-type"},"taskInfo":{"name":"accelerator-type"}},"cpu-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-cpu-limit"},"taskInfo":{"name":"cpu-limit"}},"memory-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-memory-limit"},"taskInfo":{"name":"memory-limit"}},"sum-numbers":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-limit","cpu-limit","memory-limit","accelerator-type"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}}}}' entrypoint: entrypoint podMetadata: annotations: @@ -357,15 +357,15 @@ spec: - name: component value: '{{workflow.parameters.components-49f9a898b718a077f30b7fd8c02d39767cff91ff0bbda4379daf866a91dbdb1b}}' - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-limit","accelerator-type","cpu-limit","memory-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}' + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-limit","cpu-limit","memory-limit","accelerator-type"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}' - name: container value: '{{workflow.parameters.implementations-49f9a898b718a077f30b7fd8c02d39767cff91ff0bbda4379daf866a91dbdb1b}}' - name: task-name value: sum-numbers - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - depends: accelerator-limit.Succeeded && accelerator-type.Succeeded && cpu-limit.Succeeded - && memory-limit.Succeeded + depends: accelerator-limit.Succeeded && cpu-limit.Succeeded && memory-limit.Succeeded + && accelerator-type.Succeeded name: sum-numbers-driver template: system-container-driver - arguments: From 3ba6e0b694c451100c00ef15719fe8bc0395b515 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Tue, 24 Feb 2026 01:28:02 +0300 Subject: [PATCH 28/51] - debug e2e & upgrade Signed-off-by: arpechenin --- .github/workflows/e2e-test.yml | 6 ++++++ .github/workflows/upgrade-test.yml | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 7ae220ae211..c357a3aaf34 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -160,6 +160,12 @@ jobs: tls_enabled: ${{ matrix.pod_to_pod_tls_enabled }} ca_cert_path: ${{ env.CA_CERT_PATH }} + - name: Log Argo Wf data + if: ${{ always() }} + run: | + kubectl get wf --all-namespaces + kubectl get events --all-namespaces --field-selector involvedObject.kind=Workflow --sort-by='.metadata.creationTimestamp' + end-to-end-critical-scenario-multi-user-tests: runs-on: ubuntu-latest needs: build diff --git a/.github/workflows/upgrade-test.yml b/.github/workflows/upgrade-test.yml index f6fabaed467..326d82fca9c 100644 --- a/.github/workflows/upgrade-test.yml +++ b/.github/workflows/upgrade-test.yml @@ -121,3 +121,9 @@ jobs: default_namespace: ${{ env.NAMESPACE }} python_version: ${{ env.PYTHON_VERSION }} report_name: "Upgrade Verification" + + - name: Log Argo Wf data + if: ${{ always() }} + run: | + kubectl get wf --all-namespaces + kubectl get events --all-namespaces --field-selector involvedObject.kind=Workflow --sort-by='.metadata.creationTimestamp' From 22f92702eb49e53a4063de8b0ec25fc3407b9df6 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Tue, 24 Feb 2026 21:40:51 +0300 Subject: [PATCH 29/51] - fix after merge Signed-off-by: arpechenin Signed-off-by: arpechenin --- backend/src/driver/api/request.go | 5 ++++- backend/src/driver/rpc_handler.go | 12 ++++++++++++ backend/src/v2/cmd/driver/main.go | 0 .../src/v2/compiler/argocompiler/container.go | 16 ++++++++-------- 4 files changed, 24 insertions(+), 9 deletions(-) delete mode 100644 backend/src/v2/cmd/driver/main.go diff --git a/backend/src/driver/api/request.go b/backend/src/driver/api/request.go index 60b6557792b..11942b1cbd4 100644 --- a/backend/src/driver/api/request.go +++ b/backend/src/driver/api/request.go @@ -38,7 +38,7 @@ type DriverPluginArgs struct { ExecutionIDPath string `json:"execution_id_path"` IterationCountPath string `json:"iteration_count_path"` ConditionPath string `json:"condition_path"` - PodSpecPathPath string `json:"pod_spec_patch_path"` + PodSpecPatchPath string `json:"pod_spec_patch_path"` MLMDServerAddress string `json:"mlmd_server_address"` MLMDServerPort string `json:"mlmd_server_port"` MlPipelineServerAddress string `json:"ml_pipeline_server_address"` @@ -47,6 +47,9 @@ type DriverPluginArgs struct { MetadataTLSEnabled bool `json:"metadata_tls_enabled"` CACertPath string `json:"ca_cert_path"` LogLevel string `json:"log_level"` + DefaultRunAsUser *int64 `json:"default_run_as_user,omitempty"` + DefaultRunAsGroup *int64 `json:"default_run_as_group,omitempty"` + DefaultRunAsNonRoot string `json:"default_run_as_non_root,omitempty"` } type DriverPlugin struct { diff --git a/backend/src/driver/rpc_handler.go b/backend/src/driver/rpc_handler.go index fa2e629a3cb..de50f2e7a07 100644 --- a/backend/src/driver/rpc_handler.go +++ b/backend/src/driver/rpc_handler.go @@ -244,6 +244,18 @@ func drive(args api.DriverPluginArgs) (execution *driver.Execution, err error) { case CONTAINER: options.Container = containerSpec options.KubernetesExecutorConfig = k8sExecCfg + if args.DefaultRunAsUser != nil && *args.DefaultRunAsUser >= 0 { + options.DefaultRunAsUser = args.DefaultRunAsUser + } + if args.DefaultRunAsGroup != nil && *args.DefaultRunAsGroup >= 0 { + options.DefaultRunAsGroup = args.DefaultRunAsGroup + } + if args.DefaultRunAsNonRoot != "" { + v, err := strconv.ParseBool(args.DefaultRunAsNonRoot) + if err == nil { + options.DefaultRunAsNonRoot = &v + } + } execution, driverErr = driver.Container(ctx, options, client, cacheClient) default: err = fmt.Errorf("unknown driverType %s", args.Type) diff --git a/backend/src/v2/cmd/driver/main.go b/backend/src/v2/cmd/driver/main.go deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/backend/src/v2/compiler/argocompiler/container.go b/backend/src/v2/compiler/argocompiler/container.go index 6ef53be54c3..64ecfb9fa24 100644 --- a/backend/src/v2/compiler/argocompiler/container.go +++ b/backend/src/v2/compiler/argocompiler/container.go @@ -213,19 +213,19 @@ func (c *workflowCompiler) addContainerDriverTemplate() (string, error) { if value, ok := os.LookupEnv(PublishLogsEnvVar); ok { args["publish_logs"] = value } - - containerDriverPlugin, err := driverPlugin(args) - if err != nil { - return name, fmt.Errorf("failed to add container driver plugin: %v", err) - } if c.defaultRunAsUser != nil { - args = append(args, "--default_run_as_user", strconv.FormatInt(*c.defaultRunAsUser, 10)) + args["default_run_as_user"] = strconv.FormatInt(*c.defaultRunAsUser, 10) } if c.defaultRunAsGroup != nil { - args = append(args, "--default_run_as_group", strconv.FormatInt(*c.defaultRunAsGroup, 10)) + args["default_run_as_group"] = strconv.FormatInt(*c.defaultRunAsGroup, 10) } if c.defaultRunAsNonRoot != nil { - args = append(args, "--default_run_as_non_root", strconv.FormatBool(*c.defaultRunAsNonRoot)) + args["default_run_as_non_root"] = strconv.FormatBool(*c.defaultRunAsNonRoot) + } + + containerDriverPlugin, err := driverPlugin(args) + if err != nil { + return name, fmt.Errorf("failed to add container driver plugin: %v", err) } template := &wfapi.Template{ From 804c6683636e539a075ac7646816f3edf4637ae5 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Tue, 24 Feb 2026 23:25:59 +0300 Subject: [PATCH 30/51] - reduce agent CPU requests for testing cluster Signed-off-by: arpechenin --- .../resources/manifests/base/driver-plugin-cm-path.yaml | 8 ++++---- .../standalone/tls-enabled/driver-plugin-cm-path.yaml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/resources/manifests/base/driver-plugin-cm-path.yaml b/.github/resources/manifests/base/driver-plugin-cm-path.yaml index 385c997f974..40bcc51d190 100644 --- a/.github/resources/manifests/base/driver-plugin-cm-path.yaml +++ b/.github/resources/manifests/base/driver-plugin-cm-path.yaml @@ -10,11 +10,11 @@ data: ports: - containerPort: 8080 resources: - limits: - cpu: "1" - memory: 1Gi requests: - cpu: 250m + cpu: 100m + memory: 256Mi + limits: + cpu: 500m memory: 512Mi securityContext: runAsNonRoot: true diff --git a/.github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml b/.github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml index 91117642a3d..e3ea264b4c9 100644 --- a/.github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml +++ b/.github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml @@ -10,11 +10,11 @@ data: ports: - containerPort: 8080 resources: - limits: - cpu: "1" - memory: 1Gi requests: - cpu: 250m + cpu: 100m + memory: 256Mi + limits: + cpu: 500m memory: 512Mi securityContext: runAsNonRoot: true From 91a5cd869abf582e045f4de85d28cc20afd34a62 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Wed, 25 Feb 2026 13:25:53 +0300 Subject: [PATCH 31/51] - remove debug logs - Revert memory increase (was necessary for local testing on Mac) - update check pods condition for CI Signed-off-by: arpechenin --- .../scripts/kfp-readiness/wait_for_pods.py | 3 +- .github/workflows/e2e-test.yml | 12 ------ .github/workflows/upgrade-test.yml | 6 --- backend/src/v2/compiler/argocompiler/argo.go | 2 +- test_data/compiled-workflows/add_numbers.yaml | 2 +- .../arguments_parameters.yaml | 2 +- .../compiled-workflows/artifact_cache.yaml | 2 +- .../compiled-workflows/artifact_crust.yaml | 2 +- .../compiled-workflows/artifacts_complex.yaml | 10 ++++- .../compiled-workflows/artifacts_simple.yaml | 6 ++- .../collected_artifacts.yaml | 10 ++++- .../collected_parameters.yaml | 6 ++- .../component_with_metadata_fields.yaml | 2 +- .../component_with_optional_inputs.yaml | 2 +- .../component_with_pip_index_urls.yaml | 2 +- .../component_with_pip_install.yaml | 2 +- .../component_with_pip_install_in_venv.yaml | 2 +- .../components_with_optional_artifacts.yaml | 2 +- .../compiled-workflows/concat_message.yaml | 2 +- .../conditional_producer_and_consumers.yaml | 6 ++- .../container_component_with_no_inputs.yaml | 2 +- .../compiled-workflows/container_io.yaml | 2 +- .../container_no_input.yaml | 2 +- .../container_with_artifact_output.yaml | 2 +- .../container_with_concat_placeholder.yaml | 2 +- .../container_with_if_placeholder.yaml | 2 +- ...container_with_placeholder_in_fstring.yaml | 2 +- .../containerized_python_component.yaml | 2 +- .../create_pod_metadata_complex.yaml | 6 +-- .../cross_loop_after_topology.yaml | 30 ++++++++++++- test_data/compiled-workflows/dict_input.yaml | 2 +- .../compiled-workflows/embedded_artifact.yaml | 2 +- test_data/compiled-workflows/env-var.yaml | 2 +- test_data/compiled-workflows/fail_v2.yaml | 2 +- test_data/compiled-workflows/flip_coin.yaml | 2 +- test_data/compiled-workflows/hello_world.yaml | 2 +- test_data/compiled-workflows/identity.yaml | 2 +- .../if_elif_else_complex.yaml | 10 ++++- .../if_elif_else_with_oneof_parameters.yaml | 2 +- .../if_else_with_oneof_artifacts.yaml | 2 +- .../if_else_with_oneof_parameters.yaml | 2 +- .../compiled-workflows/input_artifact.yaml | 2 +- .../iris_pipeline_compiled.yaml | 2 +- ...lightweight_python_functions_pipeline.yaml | 2 +- ...tweight_python_functions_with_outputs.yaml | 2 +- .../log_streaming_compiled.yaml | 2 +- .../compiled-workflows/long-running.yaml | 2 +- .../loop_consume_upstream.yaml | 6 ++- .../metrics_visualization_v2.yaml | 2 +- .../missing_kubernetes_optional_inputs.yaml | 2 +- .../compiled-workflows/mixed_parameters.yaml | 2 +- test_data/compiled-workflows/modelcar.yaml | 2 +- .../mounted_cabundle_configmap.yaml | 2 +- .../mounted_cabundle_secret.yaml | 2 +- .../multiple_artifacts_namedtuple.yaml | 2 +- .../multiple_parameters_namedtuple.yaml | 2 +- ...peline_opt_input_child_level_compiled.yaml | 2 +- ...sted_pipeline_opt_inputs_nil_compiled.yaml | 2 +- ...line_opt_inputs_parent_level_compiled.yaml | 2 +- .../compiled-workflows/nested_return.yaml | 2 +- .../nested_with_parameters.yaml | 10 ++++- .../notebook_component_mixed.yaml | 2 +- .../notebook_component_simple.yaml | 2 +- .../compiled-workflows/output_metrics.yaml | 2 +- .../parallel_for_after_dependency.yaml | 6 ++- .../compiled-workflows/parameter_cache.yaml | 2 +- .../compiled-workflows/parameter_oneof.yaml | 2 +- .../parameters_complex.yaml | 14 ++++++- .../compiled-workflows/parameters_simple.yaml | 6 ++- .../pipeline_as_exit_task.yaml | 2 +- .../pipeline_in_pipeline.yaml | 2 +- .../pipeline_in_pipeline_complex.yaml | 6 ++- ...pipeline_in_pipeline_loaded_from_yaml.yaml | 2 +- .../pipeline_producer_consumer.yaml | 14 ++++++- .../pipeline_with_after.yaml | 2 +- .../pipeline_with_artifact_custom_path.yaml | 2 +- ...ipeline_with_artifact_upload_download.yaml | 2 +- .../pipeline_with_concat_placeholder.yaml | 2 +- .../pipeline_with_condition.yaml | 2 +- ...namic_task_output_custom_training_job.yaml | 2 +- ...peline_with_dynamic_importer_metadata.yaml | 2 +- ...namic_task_output_custom_training_job.yaml | 2 +- .../compiled-workflows/pipeline_with_env.yaml | 2 +- .../pipeline_with_exit_handler.yaml | 2 +- .../pipeline_with_google_artifact_type.yaml | 2 +- .../pipeline_with_importer.yaml | 2 +- ...pipeline_with_importer_and_gcpc_types.yaml | 2 +- .../pipeline_with_importer_workspace.yaml | 2 +- .../pipeline_with_input_status_state.yaml | 2 +- .../pipeline_with_loops.yaml | 14 ++++++- .../pipeline_with_loops_and_conditions.yaml | 34 ++++++++++++++- .../pipeline_with_metadata_fields.yaml | 2 +- .../pipeline_with_metrics_outputs.yaml | 6 ++- .../pipeline_with_multiple_exit_handlers.yaml | 2 +- .../pipeline_with_nested_conditions.yaml | 2 +- .../pipeline_with_nested_conditions_yaml.yaml | 2 +- .../pipeline_with_nested_loops.yaml | 18 +++++++- .../pipeline_with_only_display_name.yaml | 2 +- .../pipeline_with_outputs.yaml | 2 +- ...pipeline_with_parallelfor_parallelism.yaml | 42 ++++++++++++++++++- ...ipeline_with_params_containing_format.yaml | 6 ++- .../pipeline_with_placeholders.yaml | 2 +- .../pipeline_with_pod_metadata.yaml | 10 ++--- .../pipeline_with_retry.yaml | 2 +- .../pipeline_with_reused_component.yaml | 2 +- .../pipeline_with_secret_as_env.yaml | 2 +- .../pipeline_with_secret_as_volume.yaml | 2 +- ..._string_machine_fields_pipeline_input.yaml | 2 +- ...ith_string_machine_fields_task_output.yaml | 10 ++--- .../pipeline_with_submit_request.yaml | 2 +- .../pipeline_with_task_final_status.yaml | 2 +- .../pipeline_with_task_final_status_yaml.yaml | 2 +- ...th_task_using_ignore_upstream_failure.yaml | 2 +- .../pipeline_with_utils.yaml | 2 +- .../pipeline_with_various_io_types.yaml | 2 +- .../pipeline_with_volume.yaml | 2 +- .../pipeline_with_volume_no_cache.yaml | 2 +- .../pipeline_with_workspace.yaml | 2 +- ..._with_if_placeholder_none_input_value.yaml | 2 +- test_data/compiled-workflows/preprocess.yaml | 2 +- .../producer_consumer_param_pipeline.yaml | 2 +- test_data/compiled-workflows/pvc_mount.yaml | 2 +- .../pythonic_artifact_with_single_return.yaml | 2 +- .../pythonic_artifacts_test_pipeline.yaml | 2 +- ...onic_artifacts_with_list_of_artifacts.yaml | 6 ++- ...honic_artifacts_with_multiple_returns.yaml | 2 +- .../ray_integration_compiled.yaml | 2 +- .../run_as_user_cache_disabled.yaml | 2 +- .../run_as_user_cache_enabled.yaml | 2 +- .../compiled-workflows/sequential_v1.yaml | 2 +- .../compiled-workflows/sequential_v2.yaml | 2 +- .../compiled-workflows/take_nap_compiled.yaml | 2 +- .../take_nap_pipeline_root_compiled.yaml | 2 +- .../compiled-workflows/two_step_pipeline.yaml | 2 +- .../two_step_pipeline_containerized.yaml | 2 +- .../upload_download_compiled.yaml | 2 +- .../xgboost_sample_pipeline.yaml | 2 +- 137 files changed, 370 insertions(+), 163 deletions(-) diff --git a/.github/resources/scripts/kfp-readiness/wait_for_pods.py b/.github/resources/scripts/kfp-readiness/wait_for_pods.py index fc67d5fda0e..fc604586585 100644 --- a/.github/resources/scripts/kfp-readiness/wait_for_pods.py +++ b/.github/resources/scripts/kfp-readiness/wait_for_pods.py @@ -30,7 +30,8 @@ def get_pod_statuses(): statuses = {} for pod in pods.items: pod_name = pod.metadata.name - if "system" not in pod_name: + # This filter is safe: 'ml-pipeline-persistenceagent-' will not be excluded and will be processed. + if not ("system" in pod_name or pod_name.endswith("-agent")): pod_status = pod.status.phase container_statuses = pod.status.container_statuses or [] ready = 0 diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index c357a3aaf34..f280d56ba55 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -160,12 +160,6 @@ jobs: tls_enabled: ${{ matrix.pod_to_pod_tls_enabled }} ca_cert_path: ${{ env.CA_CERT_PATH }} - - name: Log Argo Wf data - if: ${{ always() }} - run: | - kubectl get wf --all-namespaces - kubectl get events --all-namespaces --field-selector involvedObject.kind=Workflow --sort-by='.metadata.creationTimestamp' - end-to-end-critical-scenario-multi-user-tests: runs-on: ubuntu-latest needs: build @@ -259,9 +253,3 @@ jobs: python_version: ${{ env.PYTHON_VERSION }} user_namespace: ${{ env.USER_NAMESPACE }} report_name: "E2EMultiUserTests_K8s=${{ matrix.k8s_version }}_cacheEnabled=${{ matrix.cache_enabled }}_multiUser=${{ matrix.multi_user }}" - - - name: Log Argo Wf data - if: ${{ always() }} - run: | - kubectl get wf --all-namespaces - kubectl get events --all-namespaces --field-selector involvedObject.kind=Workflow --sort-by='.metadata.creationTimestamp' diff --git a/.github/workflows/upgrade-test.yml b/.github/workflows/upgrade-test.yml index 326d82fca9c..f6fabaed467 100644 --- a/.github/workflows/upgrade-test.yml +++ b/.github/workflows/upgrade-test.yml @@ -121,9 +121,3 @@ jobs: default_namespace: ${{ env.NAMESPACE }} python_version: ${{ env.PYTHON_VERSION }} report_name: "Upgrade Verification" - - - name: Log Argo Wf data - if: ${{ always() }} - run: | - kubectl get wf --all-namespaces - kubectl get events --all-namespaces --field-selector involvedObject.kind=Workflow --sort-by='.metadata.creationTimestamp' diff --git a/backend/src/v2/compiler/argocompiler/argo.go b/backend/src/v2/compiler/argocompiler/argo.go index 6328bef25c2..100201248f9 100644 --- a/backend/src/v2/compiler/argocompiler/argo.go +++ b/backend/src/v2/compiler/argocompiler/argo.go @@ -479,7 +479,7 @@ var driverResources = k8score.ResourceRequirements{ // Launcher only copies the binary into the volume, so it needs minimal resources. var launcherResources = k8score.ResourceRequirements{ Limits: map[k8score.ResourceName]k8sres.Quantity{ - k8score.ResourceMemory: k8sres.MustParse("256Mi"), + k8score.ResourceMemory: k8sres.MustParse("128Mi"), k8score.ResourceCPU: k8sres.MustParse("0.5"), }, Requests: map[k8score.ResourceName]k8sres.Quantity{ diff --git a/test_data/compiled-workflows/add_numbers.yaml b/test_data/compiled-workflows/add_numbers.yaml index 902aebbb5b4..26a9f2c0bb0 100644 --- a/test_data/compiled-workflows/add_numbers.yaml +++ b/test_data/compiled-workflows/add_numbers.yaml @@ -163,7 +163,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/arguments_parameters.yaml b/test_data/compiled-workflows/arguments_parameters.yaml index c6824d29660..7d0b50ce3ad 100644 --- a/test_data/compiled-workflows/arguments_parameters.yaml +++ b/test_data/compiled-workflows/arguments_parameters.yaml @@ -154,7 +154,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/artifact_cache.yaml b/test_data/compiled-workflows/artifact_cache.yaml index 978df7dc867..aa85fa4f3fa 100644 --- a/test_data/compiled-workflows/artifact_cache.yaml +++ b/test_data/compiled-workflows/artifact_cache.yaml @@ -182,7 +182,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/artifact_crust.yaml b/test_data/compiled-workflows/artifact_crust.yaml index 154f875b892..b4c4ba49809 100644 --- a/test_data/compiled-workflows/artifact_crust.yaml +++ b/test_data/compiled-workflows/artifact_crust.yaml @@ -182,7 +182,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/artifacts_complex.yaml b/test_data/compiled-workflows/artifacts_complex.yaml index b351d31949b..5bf2da7cab6 100644 --- a/test_data/compiled-workflows/artifacts_complex.yaml +++ b/test_data/compiled-workflows/artifacts_complex.yaml @@ -208,7 +208,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: @@ -465,6 +465,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-4"},"inputs":{"parameters":{"pipelinechannel--threshold":{"componentInputParameter":"pipelinechannel--threshold"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-3","items":{"raw":"[1, 2, 3]"}},"taskInfo":{"name":"for-loop-4"}}' + - name: task-name + value: for-loop-4 name: iteration-item-driver template: system-dag-driver - arguments: @@ -494,6 +496,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-4"},"inputs":{"parameters":{"pipelinechannel--threshold":{"componentInputParameter":"pipelinechannel--threshold"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-3","items":{"raw":"[1, 2, 3]"}},"taskInfo":{"name":"for-loop-4"}}' + - name: task-name + value: for-loop-4 name: iteration-driver template: system-dag-driver - arguments: @@ -564,6 +568,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"inputs":{"parameters":{"pipelinechannel--threshold":{"componentInputParameter":"threshold"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[1, 2, 3]"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-item-driver template: system-dag-driver - arguments: @@ -593,6 +599,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"inputs":{"parameters":{"pipelinechannel--threshold":{"componentInputParameter":"threshold"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[1, 2, 3]"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-driver template: system-dag-driver - arguments: diff --git a/test_data/compiled-workflows/artifacts_simple.yaml b/test_data/compiled-workflows/artifacts_simple.yaml index 19f5f72bb70..b67bd02a457 100644 --- a/test_data/compiled-workflows/artifacts_simple.yaml +++ b/test_data/compiled-workflows/artifacts_simple.yaml @@ -193,7 +193,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: @@ -338,6 +338,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[1, 2, 3]"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-item-driver template: system-dag-driver - arguments: @@ -367,6 +369,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[1, 2, 3]"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-driver template: system-dag-driver - arguments: diff --git a/test_data/compiled-workflows/collected_artifacts.yaml b/test_data/compiled-workflows/collected_artifacts.yaml index 285fe4ebcfa..af85c7e6605 100644 --- a/test_data/compiled-workflows/collected_artifacts.yaml +++ b/test_data/compiled-workflows/collected_artifacts.yaml @@ -274,7 +274,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: @@ -472,6 +472,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"inputs":{"parameters":{"pipelinechannel--split-chars-Output":{"componentInputParameter":"pipelinechannel--split-chars-Output"},"pipelinechannel--split-ids-Output-loop-item":{"componentInputParameter":"pipelinechannel--split-ids-Output-loop-item"}}},"parameterIterator":{"itemInput":"pipelinechannel--split-chars-Output-loop-item","items":{"inputParameter":"pipelinechannel--split-chars-Output"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-item-driver template: system-dag-driver - arguments: @@ -500,6 +502,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"inputs":{"parameters":{"pipelinechannel--split-chars-Output":{"componentInputParameter":"pipelinechannel--split-chars-Output"},"pipelinechannel--split-ids-Output-loop-item":{"componentInputParameter":"pipelinechannel--split-ids-Output-loop-item"}}},"parameterIterator":{"itemInput":"pipelinechannel--split-chars-Output-loop-item","items":{"inputParameter":"pipelinechannel--split-chars-Output"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-driver template: system-dag-driver - arguments: @@ -644,6 +648,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-1"},"dependentTasks":["split-chars","split-ids"],"inputs":{"parameters":{"pipelinechannel--split-chars-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"split-chars"}},"pipelinechannel--split-ids-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"split-ids"}}}},"parameterIterator":{"itemInput":"pipelinechannel--split-ids-Output-loop-item","items":{"inputParameter":"pipelinechannel--split-ids-Output"}},"taskInfo":{"name":"for-loop-1"}}' + - name: task-name + value: for-loop-1 name: iteration-item-driver template: system-dag-driver - arguments: @@ -672,6 +678,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-1"},"dependentTasks":["split-chars","split-ids"],"inputs":{"parameters":{"pipelinechannel--split-chars-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"split-chars"}},"pipelinechannel--split-ids-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"split-ids"}}}},"parameterIterator":{"itemInput":"pipelinechannel--split-ids-Output-loop-item","items":{"inputParameter":"pipelinechannel--split-ids-Output"}},"taskInfo":{"name":"for-loop-1"}}' + - name: task-name + value: for-loop-1 name: iteration-driver template: system-dag-driver - arguments: diff --git a/test_data/compiled-workflows/collected_parameters.yaml b/test_data/compiled-workflows/collected_parameters.yaml index cca2e227ddd..73f5b682c80 100644 --- a/test_data/compiled-workflows/collected_parameters.yaml +++ b/test_data/compiled-workflows/collected_parameters.yaml @@ -209,7 +209,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: @@ -378,6 +378,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-1"},"dependentTasks":["split-ids"],"inputs":{"parameters":{"pipelinechannel--split-ids-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"split-ids"}}}},"parameterIterator":{"itemInput":"pipelinechannel--split-ids-Output-loop-item","items":{"inputParameter":"pipelinechannel--split-ids-Output"}},"taskInfo":{"name":"for-loop-1"}}' + - name: task-name + value: for-loop-1 name: iteration-item-driver template: system-dag-driver - arguments: @@ -406,6 +408,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-1"},"dependentTasks":["split-ids"],"inputs":{"parameters":{"pipelinechannel--split-ids-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"split-ids"}}}},"parameterIterator":{"itemInput":"pipelinechannel--split-ids-Output-loop-item","items":{"inputParameter":"pipelinechannel--split-ids-Output"}},"taskInfo":{"name":"for-loop-1"}}' + - name: task-name + value: for-loop-1 name: iteration-driver template: system-dag-driver - arguments: diff --git a/test_data/compiled-workflows/component_with_metadata_fields.yaml b/test_data/compiled-workflows/component_with_metadata_fields.yaml index ed9f2fe9fd1..18361094a95 100644 --- a/test_data/compiled-workflows/component_with_metadata_fields.yaml +++ b/test_data/compiled-workflows/component_with_metadata_fields.yaml @@ -178,7 +178,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/component_with_optional_inputs.yaml b/test_data/compiled-workflows/component_with_optional_inputs.yaml index 11b2f1612f6..79bf10709d3 100644 --- a/test_data/compiled-workflows/component_with_optional_inputs.yaml +++ b/test_data/compiled-workflows/component_with_optional_inputs.yaml @@ -167,7 +167,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/component_with_pip_index_urls.yaml b/test_data/compiled-workflows/component_with_pip_index_urls.yaml index 30f3f6f4e0b..35a8514c3cd 100644 --- a/test_data/compiled-workflows/component_with_pip_index_urls.yaml +++ b/test_data/compiled-workflows/component_with_pip_index_urls.yaml @@ -165,7 +165,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/component_with_pip_install.yaml b/test_data/compiled-workflows/component_with_pip_install.yaml index 835f6584fa9..d0c656781ef 100644 --- a/test_data/compiled-workflows/component_with_pip_install.yaml +++ b/test_data/compiled-workflows/component_with_pip_install.yaml @@ -165,7 +165,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/component_with_pip_install_in_venv.yaml b/test_data/compiled-workflows/component_with_pip_install_in_venv.yaml index 72be129ed8a..34d38127f55 100644 --- a/test_data/compiled-workflows/component_with_pip_install_in_venv.yaml +++ b/test_data/compiled-workflows/component_with_pip_install_in_venv.yaml @@ -166,7 +166,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/components_with_optional_artifacts.yaml b/test_data/compiled-workflows/components_with_optional_artifacts.yaml index d6806ca1e6b..8b9bce86ee3 100644 --- a/test_data/compiled-workflows/components_with_optional_artifacts.yaml +++ b/test_data/compiled-workflows/components_with_optional_artifacts.yaml @@ -179,7 +179,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/concat_message.yaml b/test_data/compiled-workflows/concat_message.yaml index 5e6a42c0da5..d0c99d5cec6 100644 --- a/test_data/compiled-workflows/concat_message.yaml +++ b/test_data/compiled-workflows/concat_message.yaml @@ -164,7 +164,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/conditional_producer_and_consumers.yaml b/test_data/compiled-workflows/conditional_producer_and_consumers.yaml index a8a9e21815f..3d49caef58e 100644 --- a/test_data/compiled-workflows/conditional_producer_and_consumers.yaml +++ b/test_data/compiled-workflows/conditional_producer_and_consumers.yaml @@ -185,7 +185,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: @@ -393,6 +393,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"inputs":{"parameters":{"pipelinechannel--threshold":{"componentInputParameter":"threshold"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[1, 2, 3]"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-item-driver template: system-dag-driver - arguments: @@ -422,6 +424,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"inputs":{"parameters":{"pipelinechannel--threshold":{"componentInputParameter":"threshold"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[1, 2, 3]"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-driver template: system-dag-driver - arguments: diff --git a/test_data/compiled-workflows/container_component_with_no_inputs.yaml b/test_data/compiled-workflows/container_component_with_no_inputs.yaml index 09baa30b63e..d4bf7ff77ff 100644 --- a/test_data/compiled-workflows/container_component_with_no_inputs.yaml +++ b/test_data/compiled-workflows/container_component_with_no_inputs.yaml @@ -154,7 +154,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/container_io.yaml b/test_data/compiled-workflows/container_io.yaml index 2ae423682c4..6199429eee1 100644 --- a/test_data/compiled-workflows/container_io.yaml +++ b/test_data/compiled-workflows/container_io.yaml @@ -154,7 +154,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/container_no_input.yaml b/test_data/compiled-workflows/container_no_input.yaml index 041ba8af4ba..961fff99b6a 100644 --- a/test_data/compiled-workflows/container_no_input.yaml +++ b/test_data/compiled-workflows/container_no_input.yaml @@ -154,7 +154,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/container_with_artifact_output.yaml b/test_data/compiled-workflows/container_with_artifact_output.yaml index 0e521562f9e..b233f0cf7b2 100644 --- a/test_data/compiled-workflows/container_with_artifact_output.yaml +++ b/test_data/compiled-workflows/container_with_artifact_output.yaml @@ -154,7 +154,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/container_with_concat_placeholder.yaml b/test_data/compiled-workflows/container_with_concat_placeholder.yaml index 663235d2510..9c337aff62d 100644 --- a/test_data/compiled-workflows/container_with_concat_placeholder.yaml +++ b/test_data/compiled-workflows/container_with_concat_placeholder.yaml @@ -155,7 +155,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/container_with_if_placeholder.yaml b/test_data/compiled-workflows/container_with_if_placeholder.yaml index c09fdf9fee6..92673a8f2de 100644 --- a/test_data/compiled-workflows/container_with_if_placeholder.yaml +++ b/test_data/compiled-workflows/container_with_if_placeholder.yaml @@ -157,7 +157,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/container_with_placeholder_in_fstring.yaml b/test_data/compiled-workflows/container_with_placeholder_in_fstring.yaml index 05e0341960a..3e7aafb9bcc 100644 --- a/test_data/compiled-workflows/container_with_placeholder_in_fstring.yaml +++ b/test_data/compiled-workflows/container_with_placeholder_in_fstring.yaml @@ -154,7 +154,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/containerized_python_component.yaml b/test_data/compiled-workflows/containerized_python_component.yaml index 16b9841507c..2e0b2991bdf 100644 --- a/test_data/compiled-workflows/containerized_python_component.yaml +++ b/test_data/compiled-workflows/containerized_python_component.yaml @@ -154,7 +154,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/create_pod_metadata_complex.yaml b/test_data/compiled-workflows/create_pod_metadata_complex.yaml index 2fc4424f582..6c8a9880ace 100644 --- a/test_data/compiled-workflows/create_pod_metadata_complex.yaml +++ b/test_data/compiled-workflows/create_pod_metadata_complex.yaml @@ -207,7 +207,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: @@ -333,7 +333,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: @@ -464,7 +464,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/cross_loop_after_topology.yaml b/test_data/compiled-workflows/cross_loop_after_topology.yaml index cf827c9acb5..eff852c4189 100644 --- a/test_data/compiled-workflows/cross_loop_after_topology.yaml +++ b/test_data/compiled-workflows/cross_loop_after_topology.yaml @@ -184,7 +184,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: @@ -361,6 +361,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-14"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-13","items":{"raw":"[1, 2]"}},"taskInfo":{"name":"for-loop-14"}}' + - name: task-name + value: for-loop-14 name: iteration-item-driver template: system-dag-driver - arguments: @@ -390,6 +392,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-14"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-13","items":{"raw":"[1, 2]"}},"taskInfo":{"name":"for-loop-14"}}' + - name: task-name + value: for-loop-14 name: iteration-driver template: system-dag-driver - arguments: @@ -557,6 +561,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-8"},"dependentTasks":["print-op-3"],"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-7","items":{"raw":"[1, 2]"}},"taskInfo":{"name":"for-loop-8"}}' + - name: task-name + value: for-loop-8 name: iteration-item-driver template: system-dag-driver - arguments: @@ -586,6 +592,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-8"},"dependentTasks":["print-op-3"],"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-7","items":{"raw":"[1, 2]"}},"taskInfo":{"name":"for-loop-8"}}' + - name: task-name + value: for-loop-8 name: iteration-driver template: system-dag-driver - arguments: @@ -657,6 +665,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-10"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-9","items":{"raw":"[1, 2]"}},"taskInfo":{"name":"for-loop-10"}}' + - name: task-name + value: for-loop-10 name: iteration-item-driver template: system-dag-driver - arguments: @@ -686,6 +696,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-10"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-9","items":{"raw":"[1, 2]"}},"taskInfo":{"name":"for-loop-10"}}' + - name: task-name + value: for-loop-10 name: iteration-driver template: system-dag-driver - arguments: @@ -718,6 +730,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-12"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-11","items":{"raw":"[1, 2]"}},"taskInfo":{"name":"for-loop-12"}}' + - name: task-name + value: for-loop-12 name: iteration-item-driver template: system-dag-driver - arguments: @@ -747,6 +761,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-12"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-11","items":{"raw":"[1, 2]"}},"taskInfo":{"name":"for-loop-12"}}' + - name: task-name + value: for-loop-12 name: iteration-driver template: system-dag-driver - arguments: @@ -779,6 +795,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[1, 2]"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-item-driver template: system-dag-driver - arguments: @@ -808,6 +826,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[1, 2]"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-driver template: system-dag-driver - arguments: @@ -840,6 +860,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-4"},"dependentTasks":["for-loop-2"],"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-3","items":{"raw":"[1, 2]"}},"taskInfo":{"name":"for-loop-4"}}' + - name: task-name + value: for-loop-4 name: iteration-item-driver template: system-dag-driver - arguments: @@ -869,6 +891,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-4"},"dependentTasks":["for-loop-2"],"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-3","items":{"raw":"[1, 2]"}},"taskInfo":{"name":"for-loop-4"}}' + - name: task-name + value: for-loop-4 name: iteration-driver template: system-dag-driver - arguments: @@ -901,6 +925,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-6"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-5","items":{"raw":"[1, 2]"}},"taskInfo":{"name":"for-loop-6"}}' + - name: task-name + value: for-loop-6 name: iteration-item-driver template: system-dag-driver - arguments: @@ -930,6 +956,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-6"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-5","items":{"raw":"[1, 2]"}},"taskInfo":{"name":"for-loop-6"}}' + - name: task-name + value: for-loop-6 name: iteration-driver template: system-dag-driver - arguments: diff --git a/test_data/compiled-workflows/dict_input.yaml b/test_data/compiled-workflows/dict_input.yaml index a5ecd6c66bf..6f55b508c17 100644 --- a/test_data/compiled-workflows/dict_input.yaml +++ b/test_data/compiled-workflows/dict_input.yaml @@ -163,7 +163,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/embedded_artifact.yaml b/test_data/compiled-workflows/embedded_artifact.yaml index 5af008eee78..c23bd70ad5b 100644 --- a/test_data/compiled-workflows/embedded_artifact.yaml +++ b/test_data/compiled-workflows/embedded_artifact.yaml @@ -204,7 +204,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/env-var.yaml b/test_data/compiled-workflows/env-var.yaml index c49587b759f..c2100ee100a 100644 --- a/test_data/compiled-workflows/env-var.yaml +++ b/test_data/compiled-workflows/env-var.yaml @@ -165,7 +165,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/fail_v2.yaml b/test_data/compiled-workflows/fail_v2.yaml index dbf44a3cc51..ed7d088bd9c 100644 --- a/test_data/compiled-workflows/fail_v2.yaml +++ b/test_data/compiled-workflows/fail_v2.yaml @@ -163,7 +163,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/flip_coin.yaml b/test_data/compiled-workflows/flip_coin.yaml index 8c8006ad189..e71e73096d4 100644 --- a/test_data/compiled-workflows/flip_coin.yaml +++ b/test_data/compiled-workflows/flip_coin.yaml @@ -220,7 +220,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/hello_world.yaml b/test_data/compiled-workflows/hello_world.yaml index 43a7466b391..07c6f278353 100644 --- a/test_data/compiled-workflows/hello_world.yaml +++ b/test_data/compiled-workflows/hello_world.yaml @@ -154,7 +154,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/identity.yaml b/test_data/compiled-workflows/identity.yaml index f0a3f534295..b59d60161d9 100644 --- a/test_data/compiled-workflows/identity.yaml +++ b/test_data/compiled-workflows/identity.yaml @@ -163,7 +163,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/if_elif_else_complex.yaml b/test_data/compiled-workflows/if_elif_else_complex.yaml index 5f10f993fe7..8b9a58684f3 100644 --- a/test_data/compiled-workflows/if_elif_else_complex.yaml +++ b/test_data/compiled-workflows/if_elif_else_complex.yaml @@ -254,7 +254,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: @@ -464,6 +464,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-16"},"inputs":{"parameters":{"pipelinechannel--int-0-to-9999-Output":{"componentInputParameter":"pipelinechannel--int-0-to-9999-Output"},"pipelinechannel--repeat_if_lucky_number":{"componentInputParameter":"pipelinechannel--repeat_if_lucky_number"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-15","items":{"raw":"[1, 2]"}},"taskInfo":{"name":"for-loop-16"}}' + - name: task-name + value: for-loop-16 name: iteration-item-driver template: system-dag-driver - arguments: @@ -493,6 +495,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-16"},"inputs":{"parameters":{"pipelinechannel--int-0-to-9999-Output":{"componentInputParameter":"pipelinechannel--int-0-to-9999-Output"},"pipelinechannel--repeat_if_lucky_number":{"componentInputParameter":"pipelinechannel--repeat_if_lucky_number"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-15","items":{"raw":"[1, 2]"}},"taskInfo":{"name":"for-loop-16"}}' + - name: task-name + value: for-loop-16 name: iteration-driver template: system-dag-driver - arguments: @@ -1148,6 +1152,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-1"},"inputs":{"parameters":{"pipelinechannel--add_drumroll":{"componentInputParameter":"add_drumroll"},"pipelinechannel--repeat_if_lucky_number":{"componentInputParameter":"repeat_if_lucky_number"},"pipelinechannel--trials":{"componentInputParameter":"trials"}}},"parameterIterator":{"itemInput":"pipelinechannel--trials-loop-item","items":{"inputParameter":"pipelinechannel--trials"}},"taskInfo":{"name":"for-loop-1"}}' + - name: task-name + value: for-loop-1 name: iteration-item-driver template: system-dag-driver - arguments: @@ -1176,6 +1182,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-1"},"inputs":{"parameters":{"pipelinechannel--add_drumroll":{"componentInputParameter":"add_drumroll"},"pipelinechannel--repeat_if_lucky_number":{"componentInputParameter":"repeat_if_lucky_number"},"pipelinechannel--trials":{"componentInputParameter":"trials"}}},"parameterIterator":{"itemInput":"pipelinechannel--trials-loop-item","items":{"inputParameter":"pipelinechannel--trials"}},"taskInfo":{"name":"for-loop-1"}}' + - name: task-name + value: for-loop-1 name: iteration-driver template: system-dag-driver - arguments: diff --git a/test_data/compiled-workflows/if_elif_else_with_oneof_parameters.yaml b/test_data/compiled-workflows/if_elif_else_with_oneof_parameters.yaml index 9cd5108d621..1e4fc2a7315 100644 --- a/test_data/compiled-workflows/if_elif_else_with_oneof_parameters.yaml +++ b/test_data/compiled-workflows/if_elif_else_with_oneof_parameters.yaml @@ -209,7 +209,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/if_else_with_oneof_artifacts.yaml b/test_data/compiled-workflows/if_else_with_oneof_artifacts.yaml index ad1af9414fd..2aa5c9cfddc 100644 --- a/test_data/compiled-workflows/if_else_with_oneof_artifacts.yaml +++ b/test_data/compiled-workflows/if_else_with_oneof_artifacts.yaml @@ -201,7 +201,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/if_else_with_oneof_parameters.yaml b/test_data/compiled-workflows/if_else_with_oneof_parameters.yaml index d516e13ce59..5a8ff930650 100644 --- a/test_data/compiled-workflows/if_else_with_oneof_parameters.yaml +++ b/test_data/compiled-workflows/if_else_with_oneof_parameters.yaml @@ -187,7 +187,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/input_artifact.yaml b/test_data/compiled-workflows/input_artifact.yaml index f88410099a6..9fe95342e0f 100644 --- a/test_data/compiled-workflows/input_artifact.yaml +++ b/test_data/compiled-workflows/input_artifact.yaml @@ -163,7 +163,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/iris_pipeline_compiled.yaml b/test_data/compiled-workflows/iris_pipeline_compiled.yaml index ee1eb93c9be..611cdda9615 100644 --- a/test_data/compiled-workflows/iris_pipeline_compiled.yaml +++ b/test_data/compiled-workflows/iris_pipeline_compiled.yaml @@ -216,7 +216,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/lightweight_python_functions_pipeline.yaml b/test_data/compiled-workflows/lightweight_python_functions_pipeline.yaml index dbfbb57c065..543e077b70d 100644 --- a/test_data/compiled-workflows/lightweight_python_functions_pipeline.yaml +++ b/test_data/compiled-workflows/lightweight_python_functions_pipeline.yaml @@ -218,7 +218,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/lightweight_python_functions_with_outputs.yaml b/test_data/compiled-workflows/lightweight_python_functions_with_outputs.yaml index 02fc701cce2..a7864a30301 100644 --- a/test_data/compiled-workflows/lightweight_python_functions_with_outputs.yaml +++ b/test_data/compiled-workflows/lightweight_python_functions_with_outputs.yaml @@ -210,7 +210,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/log_streaming_compiled.yaml b/test_data/compiled-workflows/log_streaming_compiled.yaml index 75417b87c56..a17b7e28949 100644 --- a/test_data/compiled-workflows/log_streaming_compiled.yaml +++ b/test_data/compiled-workflows/log_streaming_compiled.yaml @@ -167,7 +167,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/long-running.yaml b/test_data/compiled-workflows/long-running.yaml index 436e40fc2bd..bfaafaf758e 100644 --- a/test_data/compiled-workflows/long-running.yaml +++ b/test_data/compiled-workflows/long-running.yaml @@ -154,7 +154,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/loop_consume_upstream.yaml b/test_data/compiled-workflows/loop_consume_upstream.yaml index 99e18f624e5..689d2b81ede 100644 --- a/test_data/compiled-workflows/loop_consume_upstream.yaml +++ b/test_data/compiled-workflows/loop_consume_upstream.yaml @@ -211,7 +211,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: @@ -382,6 +382,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-1"},"dependentTasks":["split-input"],"inputs":{"parameters":{"pipelinechannel--split-input-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"split-input"}}}},"parameterIterator":{"itemInput":"pipelinechannel--split-input-Output-loop-item","items":{"inputParameter":"pipelinechannel--split-input-Output"}},"taskInfo":{"name":"for-loop-1"}}' + - name: task-name + value: for-loop-1 name: iteration-item-driver template: system-dag-driver - arguments: @@ -410,6 +412,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-1"},"dependentTasks":["split-input"],"inputs":{"parameters":{"pipelinechannel--split-input-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"split-input"}}}},"parameterIterator":{"itemInput":"pipelinechannel--split-input-Output-loop-item","items":{"inputParameter":"pipelinechannel--split-input-Output"}},"taskInfo":{"name":"for-loop-1"}}' + - name: task-name + value: for-loop-1 name: iteration-driver template: system-dag-driver - arguments: diff --git a/test_data/compiled-workflows/metrics_visualization_v2.yaml b/test_data/compiled-workflows/metrics_visualization_v2.yaml index a32e2ce8cb6..daa771afac9 100644 --- a/test_data/compiled-workflows/metrics_visualization_v2.yaml +++ b/test_data/compiled-workflows/metrics_visualization_v2.yaml @@ -255,7 +255,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/missing_kubernetes_optional_inputs.yaml b/test_data/compiled-workflows/missing_kubernetes_optional_inputs.yaml index 22e96b994db..68a954d0461 100644 --- a/test_data/compiled-workflows/missing_kubernetes_optional_inputs.yaml +++ b/test_data/compiled-workflows/missing_kubernetes_optional_inputs.yaml @@ -166,7 +166,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/mixed_parameters.yaml b/test_data/compiled-workflows/mixed_parameters.yaml index 150d04236a8..45dae11c9f3 100644 --- a/test_data/compiled-workflows/mixed_parameters.yaml +++ b/test_data/compiled-workflows/mixed_parameters.yaml @@ -180,7 +180,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/modelcar.yaml b/test_data/compiled-workflows/modelcar.yaml index a7d0df40797..9c97b0d4818 100644 --- a/test_data/compiled-workflows/modelcar.yaml +++ b/test_data/compiled-workflows/modelcar.yaml @@ -188,7 +188,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/mounted_cabundle_configmap.yaml b/test_data/compiled-workflows/mounted_cabundle_configmap.yaml index 354b50e2b1e..f22a920bad3 100644 --- a/test_data/compiled-workflows/mounted_cabundle_configmap.yaml +++ b/test_data/compiled-workflows/mounted_cabundle_configmap.yaml @@ -157,7 +157,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/mounted_cabundle_secret.yaml b/test_data/compiled-workflows/mounted_cabundle_secret.yaml index 660c63fb0d9..6c9b8eb0721 100644 --- a/test_data/compiled-workflows/mounted_cabundle_secret.yaml +++ b/test_data/compiled-workflows/mounted_cabundle_secret.yaml @@ -157,7 +157,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/multiple_artifacts_namedtuple.yaml b/test_data/compiled-workflows/multiple_artifacts_namedtuple.yaml index df967cb979e..8e162a95168 100644 --- a/test_data/compiled-workflows/multiple_artifacts_namedtuple.yaml +++ b/test_data/compiled-workflows/multiple_artifacts_namedtuple.yaml @@ -184,7 +184,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/multiple_parameters_namedtuple.yaml b/test_data/compiled-workflows/multiple_parameters_namedtuple.yaml index c0600885313..c46a1ee2577 100644 --- a/test_data/compiled-workflows/multiple_parameters_namedtuple.yaml +++ b/test_data/compiled-workflows/multiple_parameters_namedtuple.yaml @@ -183,7 +183,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/nested_pipeline_opt_input_child_level_compiled.yaml b/test_data/compiled-workflows/nested_pipeline_opt_input_child_level_compiled.yaml index 253f3bfe06e..db766cfb3ea 100644 --- a/test_data/compiled-workflows/nested_pipeline_opt_input_child_level_compiled.yaml +++ b/test_data/compiled-workflows/nested_pipeline_opt_input_child_level_compiled.yaml @@ -243,7 +243,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml b/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml index e87ab36b379..a02c398c1b2 100644 --- a/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml +++ b/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml @@ -194,7 +194,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/nested_pipeline_opt_inputs_parent_level_compiled.yaml b/test_data/compiled-workflows/nested_pipeline_opt_inputs_parent_level_compiled.yaml index d621e217aa5..4769693969e 100644 --- a/test_data/compiled-workflows/nested_pipeline_opt_inputs_parent_level_compiled.yaml +++ b/test_data/compiled-workflows/nested_pipeline_opt_inputs_parent_level_compiled.yaml @@ -246,7 +246,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/nested_return.yaml b/test_data/compiled-workflows/nested_return.yaml index 84f3e3b4621..0e23e0ed896 100644 --- a/test_data/compiled-workflows/nested_return.yaml +++ b/test_data/compiled-workflows/nested_return.yaml @@ -164,7 +164,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/nested_with_parameters.yaml b/test_data/compiled-workflows/nested_with_parameters.yaml index 0b1345376c6..38aa8a0be0a 100644 --- a/test_data/compiled-workflows/nested_with_parameters.yaml +++ b/test_data/compiled-workflows/nested_with_parameters.yaml @@ -196,7 +196,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: @@ -390,6 +390,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-4"},"inputs":{"parameters":{"pipelinechannel--loop-item-param-1":{"componentInputParameter":"pipelinechannel--loop-item-param-1"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-3","items":{"raw":"[1, 2, 3]"}},"taskInfo":{"name":"for-loop-4"}}' + - name: task-name + value: for-loop-4 name: iteration-item-driver template: system-dag-driver - arguments: @@ -419,6 +421,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-4"},"inputs":{"parameters":{"pipelinechannel--loop-item-param-1":{"componentInputParameter":"pipelinechannel--loop-item-param-1"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-3","items":{"raw":"[1, 2, 3]"}},"taskInfo":{"name":"for-loop-4"}}' + - name: task-name + value: for-loop-4 name: iteration-driver template: system-dag-driver - arguments: @@ -465,6 +469,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[1, 2, 3]"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-item-driver template: system-dag-driver - arguments: @@ -494,6 +500,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[1, 2, 3]"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-driver template: system-dag-driver - arguments: diff --git a/test_data/compiled-workflows/notebook_component_mixed.yaml b/test_data/compiled-workflows/notebook_component_mixed.yaml index 3fcd9ced595..16e4ef6c6a4 100644 --- a/test_data/compiled-workflows/notebook_component_mixed.yaml +++ b/test_data/compiled-workflows/notebook_component_mixed.yaml @@ -392,7 +392,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/notebook_component_simple.yaml b/test_data/compiled-workflows/notebook_component_simple.yaml index 2cced3b2af7..0772e13b1d5 100644 --- a/test_data/compiled-workflows/notebook_component_simple.yaml +++ b/test_data/compiled-workflows/notebook_component_simple.yaml @@ -261,7 +261,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/output_metrics.yaml b/test_data/compiled-workflows/output_metrics.yaml index 63fea53f231..f9dbd7bc154 100644 --- a/test_data/compiled-workflows/output_metrics.yaml +++ b/test_data/compiled-workflows/output_metrics.yaml @@ -165,7 +165,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/parallel_for_after_dependency.yaml b/test_data/compiled-workflows/parallel_for_after_dependency.yaml index 61f9bc377a2..eb07ad6172a 100644 --- a/test_data/compiled-workflows/parallel_for_after_dependency.yaml +++ b/test_data/compiled-workflows/parallel_for_after_dependency.yaml @@ -166,7 +166,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: @@ -311,6 +311,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[1, 2, 3]"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-item-driver template: system-dag-driver - arguments: @@ -340,6 +342,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[1, 2, 3]"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-driver template: system-dag-driver - arguments: diff --git a/test_data/compiled-workflows/parameter_cache.yaml b/test_data/compiled-workflows/parameter_cache.yaml index 64adae7491b..13d556aeb1f 100644 --- a/test_data/compiled-workflows/parameter_cache.yaml +++ b/test_data/compiled-workflows/parameter_cache.yaml @@ -180,7 +180,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/parameter_oneof.yaml b/test_data/compiled-workflows/parameter_oneof.yaml index 161abdf2991..a77d471d93e 100644 --- a/test_data/compiled-workflows/parameter_oneof.yaml +++ b/test_data/compiled-workflows/parameter_oneof.yaml @@ -219,7 +219,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/parameters_complex.yaml b/test_data/compiled-workflows/parameters_complex.yaml index 0b768cf534a..d8d1fa9ea3b 100644 --- a/test_data/compiled-workflows/parameters_complex.yaml +++ b/test_data/compiled-workflows/parameters_complex.yaml @@ -213,7 +213,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: @@ -358,6 +358,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-4"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-3","items":{"raw":"[4, 5, 6]"}},"taskInfo":{"name":"for-loop-4"}}' + - name: task-name + value: for-loop-4 name: iteration-item-driver template: system-dag-driver - arguments: @@ -387,6 +389,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-4"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-3","items":{"raw":"[4, 5, 6]"}},"taskInfo":{"name":"for-loop-4"}}' + - name: task-name + value: for-loop-4 name: iteration-driver template: system-dag-driver - arguments: @@ -538,6 +542,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[1, 2, 3]"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-item-driver template: system-dag-driver - arguments: @@ -567,6 +573,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[1, 2, 3]"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-driver template: system-dag-driver - arguments: @@ -599,6 +607,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-6"},"dependentTasks":["for-loop-2"],"inputs":{"parameters":{"pipelinechannel--for-loop-2-pipelinechannel--double-2-Output":{"taskOutputParameter":{"outputParameterKey":"pipelinechannel--double-2-Output","producerTask":"for-loop-2"}},"pipelinechannel--for-loop-2-pipelinechannel--double-Output":{"taskOutputParameter":{"outputParameterKey":"pipelinechannel--double-Output","producerTask":"for-loop-2"}}}},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-5","items":{"raw":"[0, 0, 0]"}},"taskInfo":{"name":"for-loop-6"}}' + - name: task-name + value: for-loop-6 name: iteration-item-driver template: system-dag-driver - arguments: @@ -628,6 +638,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-6"},"dependentTasks":["for-loop-2"],"inputs":{"parameters":{"pipelinechannel--for-loop-2-pipelinechannel--double-2-Output":{"taskOutputParameter":{"outputParameterKey":"pipelinechannel--double-2-Output","producerTask":"for-loop-2"}},"pipelinechannel--for-loop-2-pipelinechannel--double-Output":{"taskOutputParameter":{"outputParameterKey":"pipelinechannel--double-Output","producerTask":"for-loop-2"}}}},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-5","items":{"raw":"[0, 0, 0]"}},"taskInfo":{"name":"for-loop-6"}}' + - name: task-name + value: for-loop-6 name: iteration-driver template: system-dag-driver - arguments: diff --git a/test_data/compiled-workflows/parameters_simple.yaml b/test_data/compiled-workflows/parameters_simple.yaml index 08cc9c1766c..63cbe75e9a9 100644 --- a/test_data/compiled-workflows/parameters_simple.yaml +++ b/test_data/compiled-workflows/parameters_simple.yaml @@ -184,7 +184,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: @@ -329,6 +329,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[1, 2, 3]"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-item-driver template: system-dag-driver - arguments: @@ -358,6 +360,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[1, 2, 3]"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-driver template: system-dag-driver - arguments: diff --git a/test_data/compiled-workflows/pipeline_as_exit_task.yaml b/test_data/compiled-workflows/pipeline_as_exit_task.yaml index e8920879925..dc05bdfc6b7 100644 --- a/test_data/compiled-workflows/pipeline_as_exit_task.yaml +++ b/test_data/compiled-workflows/pipeline_as_exit_task.yaml @@ -200,7 +200,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_in_pipeline.yaml b/test_data/compiled-workflows/pipeline_in_pipeline.yaml index f2ef62dca8b..025c22292e5 100644 --- a/test_data/compiled-workflows/pipeline_in_pipeline.yaml +++ b/test_data/compiled-workflows/pipeline_in_pipeline.yaml @@ -169,7 +169,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml b/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml index cd56cf69db9..9c87c807b08 100644 --- a/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml +++ b/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml @@ -178,7 +178,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: @@ -464,6 +464,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[\"Hello\", \"world!\"]"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-item-driver template: system-dag-driver - arguments: @@ -493,6 +495,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[\"Hello\", \"world!\"]"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-driver template: system-dag-driver - arguments: diff --git a/test_data/compiled-workflows/pipeline_in_pipeline_loaded_from_yaml.yaml b/test_data/compiled-workflows/pipeline_in_pipeline_loaded_from_yaml.yaml index e0c5f98f183..4c06eb74de9 100644 --- a/test_data/compiled-workflows/pipeline_in_pipeline_loaded_from_yaml.yaml +++ b/test_data/compiled-workflows/pipeline_in_pipeline_loaded_from_yaml.yaml @@ -185,7 +185,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_producer_consumer.yaml b/test_data/compiled-workflows/pipeline_producer_consumer.yaml index 46e7800a487..96968191410 100644 --- a/test_data/compiled-workflows/pipeline_producer_consumer.yaml +++ b/test_data/compiled-workflows/pipeline_producer_consumer.yaml @@ -216,7 +216,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: @@ -361,6 +361,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-2-2"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[\"m\", \"a\", \"t\", \"h\"]"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-item-driver template: system-dag-driver - arguments: @@ -390,6 +392,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-2-2"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[\"m\", \"a\", \"t\", \"h\"]"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-driver template: system-dag-driver - arguments: @@ -521,6 +525,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-4"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-3","items":{"raw":"[1, 2, 3]"}},"taskInfo":{"name":"for-loop-4"}}' + - name: task-name + value: for-loop-4 name: iteration-item-driver template: system-dag-driver - arguments: @@ -550,6 +556,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-4"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-3","items":{"raw":"[1, 2, 3]"}},"taskInfo":{"name":"for-loop-4"}}' + - name: task-name + value: for-loop-4 name: iteration-driver template: system-dag-driver - arguments: @@ -596,6 +604,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[1, 2, 3]"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-item-driver template: system-dag-driver - arguments: @@ -625,6 +635,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[1, 2, 3]"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-driver template: system-dag-driver - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_after.yaml b/test_data/compiled-workflows/pipeline_with_after.yaml index dc7acde896f..82e697785c1 100644 --- a/test_data/compiled-workflows/pipeline_with_after.yaml +++ b/test_data/compiled-workflows/pipeline_with_after.yaml @@ -157,7 +157,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_artifact_custom_path.yaml b/test_data/compiled-workflows/pipeline_with_artifact_custom_path.yaml index f34f2413c0e..6d3fce0d341 100644 --- a/test_data/compiled-workflows/pipeline_with_artifact_custom_path.yaml +++ b/test_data/compiled-workflows/pipeline_with_artifact_custom_path.yaml @@ -185,7 +185,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_artifact_upload_download.yaml b/test_data/compiled-workflows/pipeline_with_artifact_upload_download.yaml index 5bf452566ce..b2a91f9dd6c 100644 --- a/test_data/compiled-workflows/pipeline_with_artifact_upload_download.yaml +++ b/test_data/compiled-workflows/pipeline_with_artifact_upload_download.yaml @@ -184,7 +184,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_concat_placeholder.yaml b/test_data/compiled-workflows/pipeline_with_concat_placeholder.yaml index 9337c16c7ce..22ae7858cb4 100644 --- a/test_data/compiled-workflows/pipeline_with_concat_placeholder.yaml +++ b/test_data/compiled-workflows/pipeline_with_concat_placeholder.yaml @@ -156,7 +156,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_condition.yaml b/test_data/compiled-workflows/pipeline_with_condition.yaml index 8b4ee02814c..e06c6fef231 100644 --- a/test_data/compiled-workflows/pipeline_with_condition.yaml +++ b/test_data/compiled-workflows/pipeline_with_condition.yaml @@ -182,7 +182,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml b/test_data/compiled-workflows/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml index 63a5de13675..989290e9ce2 100644 --- a/test_data/compiled-workflows/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml +++ b/test_data/compiled-workflows/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml @@ -279,7 +279,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_dynamic_importer_metadata.yaml b/test_data/compiled-workflows/pipeline_with_dynamic_importer_metadata.yaml index 64e4298f07f..800661365f2 100644 --- a/test_data/compiled-workflows/pipeline_with_dynamic_importer_metadata.yaml +++ b/test_data/compiled-workflows/pipeline_with_dynamic_importer_metadata.yaml @@ -240,7 +240,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_dynamic_task_output_custom_training_job.yaml b/test_data/compiled-workflows/pipeline_with_dynamic_task_output_custom_training_job.yaml index 5a77fdbc063..ca70715b017 100644 --- a/test_data/compiled-workflows/pipeline_with_dynamic_task_output_custom_training_job.yaml +++ b/test_data/compiled-workflows/pipeline_with_dynamic_task_output_custom_training_job.yaml @@ -241,7 +241,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_env.yaml b/test_data/compiled-workflows/pipeline_with_env.yaml index dd841a15348..4d3b51c6ac3 100644 --- a/test_data/compiled-workflows/pipeline_with_env.yaml +++ b/test_data/compiled-workflows/pipeline_with_env.yaml @@ -169,7 +169,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_exit_handler.yaml b/test_data/compiled-workflows/pipeline_with_exit_handler.yaml index 604ae68924b..5cfebf6fdae 100644 --- a/test_data/compiled-workflows/pipeline_with_exit_handler.yaml +++ b/test_data/compiled-workflows/pipeline_with_exit_handler.yaml @@ -181,7 +181,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_google_artifact_type.yaml b/test_data/compiled-workflows/pipeline_with_google_artifact_type.yaml index 0be0ff47892..b366961be53 100644 --- a/test_data/compiled-workflows/pipeline_with_google_artifact_type.yaml +++ b/test_data/compiled-workflows/pipeline_with_google_artifact_type.yaml @@ -258,7 +258,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_importer.yaml b/test_data/compiled-workflows/pipeline_with_importer.yaml index 3af33e3be15..79d2c3afa58 100644 --- a/test_data/compiled-workflows/pipeline_with_importer.yaml +++ b/test_data/compiled-workflows/pipeline_with_importer.yaml @@ -248,7 +248,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_importer_and_gcpc_types.yaml b/test_data/compiled-workflows/pipeline_with_importer_and_gcpc_types.yaml index a9af7bb8798..94c4f279f2c 100644 --- a/test_data/compiled-workflows/pipeline_with_importer_and_gcpc_types.yaml +++ b/test_data/compiled-workflows/pipeline_with_importer_and_gcpc_types.yaml @@ -158,7 +158,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_importer_workspace.yaml b/test_data/compiled-workflows/pipeline_with_importer_workspace.yaml index 29ac513e0ba..89df00e3c37 100644 --- a/test_data/compiled-workflows/pipeline_with_importer_workspace.yaml +++ b/test_data/compiled-workflows/pipeline_with_importer_workspace.yaml @@ -322,7 +322,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_input_status_state.yaml b/test_data/compiled-workflows/pipeline_with_input_status_state.yaml index fde4b1105de..c9198cc09fc 100644 --- a/test_data/compiled-workflows/pipeline_with_input_status_state.yaml +++ b/test_data/compiled-workflows/pipeline_with_input_status_state.yaml @@ -181,7 +181,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_loops.yaml b/test_data/compiled-workflows/pipeline_with_loops.yaml index f31b91a63de..adf9ab8a685 100644 --- a/test_data/compiled-workflows/pipeline_with_loops.yaml +++ b/test_data/compiled-workflows/pipeline_with_loops.yaml @@ -197,7 +197,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: @@ -501,6 +501,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-1"},"inputs":{"parameters":{"pipelinechannel--loop_parameter":{"componentInputParameter":"loop_parameter"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop_parameter-loop-item","items":{"inputParameter":"pipelinechannel--loop_parameter"}},"taskInfo":{"name":"for-loop-1"}}' + - name: task-name + value: for-loop-1 name: iteration-item-driver template: system-dag-driver - arguments: @@ -529,6 +531,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-1"},"inputs":{"parameters":{"pipelinechannel--loop_parameter":{"componentInputParameter":"loop_parameter"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop_parameter-loop-item","items":{"inputParameter":"pipelinechannel--loop_parameter"}},"taskInfo":{"name":"for-loop-1"}}' + - name: task-name + value: for-loop-1 name: iteration-driver template: system-dag-driver - arguments: @@ -560,6 +564,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"dependentTasks":["args-generator-op"],"inputs":{"parameters":{"pipelinechannel--args-generator-op-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"args-generator-op"}}}},"parameterIterator":{"itemInput":"pipelinechannel--args-generator-op-Output-loop-item","items":{"inputParameter":"pipelinechannel--args-generator-op-Output"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-item-driver template: system-dag-driver - arguments: @@ -588,6 +594,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"dependentTasks":["args-generator-op"],"inputs":{"parameters":{"pipelinechannel--args-generator-op-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"args-generator-op"}}}},"parameterIterator":{"itemInput":"pipelinechannel--args-generator-op-Output-loop-item","items":{"inputParameter":"pipelinechannel--args-generator-op-Output"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-driver template: system-dag-driver - arguments: @@ -620,6 +628,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-4"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-3","items":{"raw":"[{\"A_a\": \"1\", \"B_b\": \"2\"}, {\"A_a\": \"10\", \"B_b\": \"20\"}]"}},"taskInfo":{"name":"for-loop-4"}}' + - name: task-name + value: for-loop-4 name: iteration-item-driver template: system-dag-driver - arguments: @@ -649,6 +659,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-4"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-3","items":{"raw":"[{\"A_a\": \"1\", \"B_b\": \"2\"}, {\"A_a\": \"10\", \"B_b\": \"20\"}]"}},"taskInfo":{"name":"for-loop-4"}}' + - name: task-name + value: for-loop-4 name: iteration-driver template: system-dag-driver - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_loops_and_conditions.yaml b/test_data/compiled-workflows/pipeline_with_loops_and_conditions.yaml index 1fe33c8ba53..63d30ff1735 100644 --- a/test_data/compiled-workflows/pipeline_with_loops_and_conditions.yaml +++ b/test_data/compiled-workflows/pipeline_with_loops_and_conditions.yaml @@ -243,7 +243,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: @@ -452,6 +452,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-7"},"inputs":{"parameters":{"pipelinechannel--args-generator-op-Output-loop-item":{"componentInputParameter":"pipelinechannel--args-generator-op-Output-loop-item"},"pipelinechannel--flip-coin-op-Output":{"componentInputParameter":"pipelinechannel--flip-coin-op-Output"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-6","items":{"raw":"[{\"a\": \"-1\"}, {\"a\": \"-2\"}]"}},"taskInfo":{"name":"for-loop-7"}}' + - name: task-name + value: for-loop-7 name: iteration-item-driver template: system-dag-driver - arguments: @@ -481,6 +483,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-7"},"inputs":{"parameters":{"pipelinechannel--args-generator-op-Output-loop-item":{"componentInputParameter":"pipelinechannel--args-generator-op-Output-loop-item"},"pipelinechannel--flip-coin-op-Output":{"componentInputParameter":"pipelinechannel--flip-coin-op-Output"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-6","items":{"raw":"[{\"a\": \"-1\"}, {\"a\": \"-2\"}]"}},"taskInfo":{"name":"for-loop-7"}}' + - name: task-name + value: for-loop-7 name: iteration-driver template: system-dag-driver - arguments: @@ -677,6 +681,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-10"},"inputs":{"parameters":{"pipelinechannel--args-generator-op-2-Output":{"componentInputParameter":"pipelinechannel--args-generator-op-2-Output"},"pipelinechannel--flip-coin-op-Output":{"componentInputParameter":"pipelinechannel--flip-coin-op-Output"},"pipelinechannel--loop_parameter-loop-item":{"componentInputParameter":"pipelinechannel--loop_parameter-loop-item"}}},"parameterIterator":{"itemInput":"pipelinechannel--args-generator-op-2-Output-loop-item","items":{"inputParameter":"pipelinechannel--args-generator-op-2-Output"}},"taskInfo":{"name":"for-loop-10"}}' + - name: task-name + value: for-loop-10 name: iteration-item-driver template: system-dag-driver - arguments: @@ -705,6 +711,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-10"},"inputs":{"parameters":{"pipelinechannel--args-generator-op-2-Output":{"componentInputParameter":"pipelinechannel--args-generator-op-2-Output"},"pipelinechannel--flip-coin-op-Output":{"componentInputParameter":"pipelinechannel--flip-coin-op-Output"},"pipelinechannel--loop_parameter-loop-item":{"componentInputParameter":"pipelinechannel--loop_parameter-loop-item"}}},"parameterIterator":{"itemInput":"pipelinechannel--args-generator-op-2-Output-loop-item","items":{"inputParameter":"pipelinechannel--args-generator-op-2-Output"}},"taskInfo":{"name":"for-loop-10"}}' + - name: task-name + value: for-loop-10 name: iteration-driver template: system-dag-driver - arguments: @@ -775,6 +783,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-12"},"inputs":{"parameters":{"pipelinechannel--flip-coin-op-Output":{"componentInputParameter":"pipelinechannel--flip-coin-op-Output"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-11","items":{"raw":"[\"1\", \"2\"]"}},"taskInfo":{"name":"for-loop-12"}}' + - name: task-name + value: for-loop-12 name: iteration-item-driver template: system-dag-driver - arguments: @@ -804,6 +814,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-12"},"inputs":{"parameters":{"pipelinechannel--flip-coin-op-Output":{"componentInputParameter":"pipelinechannel--flip-coin-op-Output"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-11","items":{"raw":"[\"1\", \"2\"]"}},"taskInfo":{"name":"for-loop-12"}}' + - name: task-name + value: for-loop-12 name: iteration-driver template: system-dag-driver - arguments: @@ -835,6 +847,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-8"},"inputs":{"parameters":{"pipelinechannel--args-generator-op-Output-loop-item":{"componentInputParameter":"pipelinechannel--args-generator-op-Output-loop-item"},"pipelinechannel--args-generator-op-Output-loop-item-subvar-B_b":{"componentInputParameter":"pipelinechannel--args-generator-op-Output-loop-item","parameterExpressionSelector":"parseJson(string_value)[\"B_b\"]"},"pipelinechannel--flip-coin-op-Output":{"componentInputParameter":"pipelinechannel--flip-coin-op-Output"}}},"parameterIterator":{"itemInput":"pipelinechannel--args-generator-op-Output-loop-item-subvar-B_b-loop-item","items":{"inputParameter":"pipelinechannel--args-generator-op-Output-loop-item-subvar-B_b"}},"taskInfo":{"name":"for-loop-8"}}' + - name: task-name + value: for-loop-8 name: iteration-item-driver template: system-dag-driver - arguments: @@ -863,6 +877,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-8"},"inputs":{"parameters":{"pipelinechannel--args-generator-op-Output-loop-item":{"componentInputParameter":"pipelinechannel--args-generator-op-Output-loop-item"},"pipelinechannel--args-generator-op-Output-loop-item-subvar-B_b":{"componentInputParameter":"pipelinechannel--args-generator-op-Output-loop-item","parameterExpressionSelector":"parseJson(string_value)[\"B_b\"]"},"pipelinechannel--flip-coin-op-Output":{"componentInputParameter":"pipelinechannel--flip-coin-op-Output"}}},"parameterIterator":{"itemInput":"pipelinechannel--args-generator-op-Output-loop-item-subvar-B_b-loop-item","items":{"inputParameter":"pipelinechannel--args-generator-op-Output-loop-item-subvar-B_b"}},"taskInfo":{"name":"for-loop-8"}}' + - name: task-name + value: for-loop-8 name: iteration-driver template: system-dag-driver - arguments: @@ -894,6 +910,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-9"},"inputs":{"parameters":{"pipelinechannel--args-generator-op-2-Output":{"componentInputParameter":"pipelinechannel--args-generator-op-2-Output"},"pipelinechannel--flip-coin-op-Output":{"componentInputParameter":"pipelinechannel--flip-coin-op-Output"},"pipelinechannel--loop_parameter":{"componentInputParameter":"pipelinechannel--loop_parameter"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop_parameter-loop-item","items":{"inputParameter":"pipelinechannel--loop_parameter"}},"taskInfo":{"name":"for-loop-9"}}' + - name: task-name + value: for-loop-9 name: iteration-item-driver template: system-dag-driver - arguments: @@ -922,6 +940,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-9"},"inputs":{"parameters":{"pipelinechannel--args-generator-op-2-Output":{"componentInputParameter":"pipelinechannel--args-generator-op-2-Output"},"pipelinechannel--flip-coin-op-Output":{"componentInputParameter":"pipelinechannel--flip-coin-op-Output"},"pipelinechannel--loop_parameter":{"componentInputParameter":"pipelinechannel--loop_parameter"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop_parameter-loop-item","items":{"inputParameter":"pipelinechannel--loop_parameter"}},"taskInfo":{"name":"for-loop-9"}}' + - name: task-name + value: for-loop-9 name: iteration-driver template: system-dag-driver - arguments: @@ -1072,6 +1092,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"dependentTasks":["args-generator-op-2"],"inputs":{"parameters":{"pipelinechannel--args-generator-op-2-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"args-generator-op-2"}},"pipelinechannel--args-generator-op-Output":{"componentInputParameter":"pipelinechannel--args-generator-op-Output"},"pipelinechannel--flip-coin-op-Output":{"componentInputParameter":"pipelinechannel--flip-coin-op-Output"},"pipelinechannel--loop_parameter":{"componentInputParameter":"pipelinechannel--loop_parameter"},"pipelinechannel--msg":{"componentInputParameter":"pipelinechannel--msg"}}},"parameterIterator":{"itemInput":"pipelinechannel--args-generator-op-Output-loop-item","items":{"inputParameter":"pipelinechannel--args-generator-op-Output"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-item-driver template: system-dag-driver - arguments: @@ -1100,6 +1122,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"dependentTasks":["args-generator-op-2"],"inputs":{"parameters":{"pipelinechannel--args-generator-op-2-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"args-generator-op-2"}},"pipelinechannel--args-generator-op-Output":{"componentInputParameter":"pipelinechannel--args-generator-op-Output"},"pipelinechannel--flip-coin-op-Output":{"componentInputParameter":"pipelinechannel--flip-coin-op-Output"},"pipelinechannel--loop_parameter":{"componentInputParameter":"pipelinechannel--loop_parameter"},"pipelinechannel--msg":{"componentInputParameter":"pipelinechannel--msg"}}},"parameterIterator":{"itemInput":"pipelinechannel--args-generator-op-Output-loop-item","items":{"inputParameter":"pipelinechannel--args-generator-op-Output"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-driver template: system-dag-driver - arguments: @@ -1202,6 +1226,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-16"},"inputs":{"parameters":{"pipelinechannel--loop_parameter-loop-item":{"componentInputParameter":"pipelinechannel--loop_parameter-loop-item"},"pipelinechannel--loop_parameter-loop-item-subvar-B_b":{"componentInputParameter":"pipelinechannel--loop_parameter-loop-item","parameterExpressionSelector":"parseJson(string_value)[\"B_b\"]"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop_parameter-loop-item-subvar-B_b-loop-item","items":{"inputParameter":"pipelinechannel--loop_parameter-loop-item-subvar-B_b"}},"taskInfo":{"name":"for-loop-16"}}' + - name: task-name + value: for-loop-16 name: iteration-item-driver template: system-dag-driver - arguments: @@ -1230,6 +1256,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-16"},"inputs":{"parameters":{"pipelinechannel--loop_parameter-loop-item":{"componentInputParameter":"pipelinechannel--loop_parameter-loop-item"},"pipelinechannel--loop_parameter-loop-item-subvar-B_b":{"componentInputParameter":"pipelinechannel--loop_parameter-loop-item","parameterExpressionSelector":"parseJson(string_value)[\"B_b\"]"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop_parameter-loop-item-subvar-B_b-loop-item","items":{"inputParameter":"pipelinechannel--loop_parameter-loop-item-subvar-B_b"}},"taskInfo":{"name":"for-loop-16"}}' + - name: task-name + value: for-loop-16 name: iteration-driver template: system-dag-driver - arguments: @@ -1306,6 +1334,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-14"},"inputs":{"parameters":{"pipelinechannel--loop_parameter":{"componentInputParameter":"loop_parameter"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop_parameter-loop-item","items":{"inputParameter":"pipelinechannel--loop_parameter"}},"taskInfo":{"name":"for-loop-14"}}' + - name: task-name + value: for-loop-14 name: iteration-item-driver template: system-dag-driver - arguments: @@ -1334,6 +1364,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-14"},"inputs":{"parameters":{"pipelinechannel--loop_parameter":{"componentInputParameter":"loop_parameter"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop_parameter-loop-item","items":{"inputParameter":"pipelinechannel--loop_parameter"}},"taskInfo":{"name":"for-loop-14"}}' + - name: task-name + value: for-loop-14 name: iteration-driver template: system-dag-driver - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_metadata_fields.yaml b/test_data/compiled-workflows/pipeline_with_metadata_fields.yaml index 421f17ee670..ca2dead409e 100644 --- a/test_data/compiled-workflows/pipeline_with_metadata_fields.yaml +++ b/test_data/compiled-workflows/pipeline_with_metadata_fields.yaml @@ -194,7 +194,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_metrics_outputs.yaml b/test_data/compiled-workflows/pipeline_with_metrics_outputs.yaml index 69adb3797f8..e202fabc5c3 100644 --- a/test_data/compiled-workflows/pipeline_with_metrics_outputs.yaml +++ b/test_data/compiled-workflows/pipeline_with_metrics_outputs.yaml @@ -168,7 +168,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: @@ -313,6 +313,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[1, 2]"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-item-driver template: system-dag-driver - arguments: @@ -342,6 +344,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[1, 2]"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-driver template: system-dag-driver - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_multiple_exit_handlers.yaml b/test_data/compiled-workflows/pipeline_with_multiple_exit_handlers.yaml index 8b0a9f97198..cbed20cb632 100644 --- a/test_data/compiled-workflows/pipeline_with_multiple_exit_handlers.yaml +++ b/test_data/compiled-workflows/pipeline_with_multiple_exit_handlers.yaml @@ -187,7 +187,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_nested_conditions.yaml b/test_data/compiled-workflows/pipeline_with_nested_conditions.yaml index 7d9cc3db54c..60caddfa9de 100644 --- a/test_data/compiled-workflows/pipeline_with_nested_conditions.yaml +++ b/test_data/compiled-workflows/pipeline_with_nested_conditions.yaml @@ -184,7 +184,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_nested_conditions_yaml.yaml b/test_data/compiled-workflows/pipeline_with_nested_conditions_yaml.yaml index 1f910249bc2..1162107adbf 100644 --- a/test_data/compiled-workflows/pipeline_with_nested_conditions_yaml.yaml +++ b/test_data/compiled-workflows/pipeline_with_nested_conditions_yaml.yaml @@ -196,7 +196,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_nested_loops.yaml b/test_data/compiled-workflows/pipeline_with_nested_loops.yaml index dbc04c58adb..09292a9d078 100644 --- a/test_data/compiled-workflows/pipeline_with_nested_loops.yaml +++ b/test_data/compiled-workflows/pipeline_with_nested_loops.yaml @@ -174,7 +174,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: @@ -318,6 +318,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"inputs":{"parameters":{"pipelinechannel--loop_parameter-loop-item":{"componentInputParameter":"pipelinechannel--loop_parameter-loop-item"},"pipelinechannel--loop_parameter-loop-item-subvar-p_a":{"componentInputParameter":"pipelinechannel--loop_parameter-loop-item","parameterExpressionSelector":"parseJson(string_value)[\"p_a\"]"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop_parameter-loop-item-subvar-p_a-loop-item","items":{"inputParameter":"pipelinechannel--loop_parameter-loop-item-subvar-p_a"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-item-driver template: system-dag-driver - arguments: @@ -346,6 +348,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"inputs":{"parameters":{"pipelinechannel--loop_parameter-loop-item":{"componentInputParameter":"pipelinechannel--loop_parameter-loop-item"},"pipelinechannel--loop_parameter-loop-item-subvar-p_a":{"componentInputParameter":"pipelinechannel--loop_parameter-loop-item","parameterExpressionSelector":"parseJson(string_value)[\"p_a\"]"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop_parameter-loop-item-subvar-p_a-loop-item","items":{"inputParameter":"pipelinechannel--loop_parameter-loop-item-subvar-p_a"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-driver template: system-dag-driver - arguments: @@ -424,6 +428,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-6"},"inputs":{"parameters":{"pipelinechannel--loop-item-param-3":{"componentInputParameter":"pipelinechannel--loop-item-param-3"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-5","items":{"raw":"[\"100\", \"200\", \"300\"]"}},"taskInfo":{"name":"for-loop-6"}}' + - name: task-name + value: for-loop-6 name: iteration-item-driver template: system-dag-driver - arguments: @@ -453,6 +459,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-6"},"inputs":{"parameters":{"pipelinechannel--loop-item-param-3":{"componentInputParameter":"pipelinechannel--loop-item-param-3"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-5","items":{"raw":"[\"100\", \"200\", \"300\"]"}},"taskInfo":{"name":"for-loop-6"}}' + - name: task-name + value: for-loop-6 name: iteration-driver template: system-dag-driver - arguments: @@ -522,6 +530,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-1"},"inputs":{"parameters":{"pipelinechannel--loop_parameter":{"componentInputParameter":"loop_parameter"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop_parameter-loop-item","items":{"inputParameter":"pipelinechannel--loop_parameter"}},"taskInfo":{"name":"for-loop-1"}}' + - name: task-name + value: for-loop-1 name: iteration-item-driver template: system-dag-driver - arguments: @@ -550,6 +560,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-1"},"inputs":{"parameters":{"pipelinechannel--loop_parameter":{"componentInputParameter":"loop_parameter"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop_parameter-loop-item","items":{"inputParameter":"pipelinechannel--loop_parameter"}},"taskInfo":{"name":"for-loop-1"}}' + - name: task-name + value: for-loop-1 name: iteration-driver template: system-dag-driver - arguments: @@ -582,6 +594,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-4"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-3","items":{"raw":"[\"1\", \"2\"]"}},"taskInfo":{"name":"for-loop-4"}}' + - name: task-name + value: for-loop-4 name: iteration-item-driver template: system-dag-driver - arguments: @@ -611,6 +625,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-4"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-3","items":{"raw":"[\"1\", \"2\"]"}},"taskInfo":{"name":"for-loop-4"}}' + - name: task-name + value: for-loop-4 name: iteration-driver template: system-dag-driver - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_only_display_name.yaml b/test_data/compiled-workflows/pipeline_with_only_display_name.yaml index add6d913166..6f18284804d 100644 --- a/test_data/compiled-workflows/pipeline_with_only_display_name.yaml +++ b/test_data/compiled-workflows/pipeline_with_only_display_name.yaml @@ -154,7 +154,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_outputs.yaml b/test_data/compiled-workflows/pipeline_with_outputs.yaml index 832e2226059..d1c175a3f22 100644 --- a/test_data/compiled-workflows/pipeline_with_outputs.yaml +++ b/test_data/compiled-workflows/pipeline_with_outputs.yaml @@ -170,7 +170,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_parallelfor_parallelism.yaml b/test_data/compiled-workflows/pipeline_with_parallelfor_parallelism.yaml index 65d909d2f56..4cda2cd545f 100644 --- a/test_data/compiled-workflows/pipeline_with_parallelfor_parallelism.yaml +++ b/test_data/compiled-workflows/pipeline_with_parallelfor_parallelism.yaml @@ -261,7 +261,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: @@ -405,6 +405,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"inputs":{"parameters":{"pipelinechannel--loop_parameter":{"componentInputParameter":"pipelinechannel--loop_parameter"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop_parameter-loop-item","items":{"inputParameter":"pipelinechannel--loop_parameter"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-item-driver template: system-dag-driver - arguments: @@ -433,6 +435,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"inputs":{"parameters":{"pipelinechannel--loop_parameter":{"componentInputParameter":"pipelinechannel--loop_parameter"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop_parameter-loop-item","items":{"inputParameter":"pipelinechannel--loop_parameter"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-driver template: system-dag-driver - arguments: @@ -687,6 +691,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-6"},"iteratorPolicy":{"parallelismLimit":1},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-5","items":{"raw":"[{\"A_a\": \"10\", \"B_b\": \"20\"}, {\"A_a\": \"100\", \"B_b\": \"200\"}]"}},"taskInfo":{"name":"for-loop-6"}}' + - name: task-name + value: for-loop-6 name: iteration-item-driver template: system-dag-driver - arguments: @@ -716,6 +722,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-6"},"iteratorPolicy":{"parallelismLimit":1},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-5","items":{"raw":"[{\"A_a\": \"10\", \"B_b\": \"20\"}, {\"A_a\": \"100\", \"B_b\": \"200\"}]"}},"taskInfo":{"name":"for-loop-6"}}' + - name: task-name + value: for-loop-6 name: iteration-driver template: system-dag-driver - arguments: @@ -874,6 +882,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-1"},"inputs":{"parameters":{"pipelinechannel--loop_parameter":{"componentInputParameter":"loop_parameter"}}},"iteratorPolicy":{"parallelismLimit":2},"parameterIterator":{"itemInput":"pipelinechannel--loop_parameter-loop-item","items":{"inputParameter":"pipelinechannel--loop_parameter"}},"taskInfo":{"name":"for-loop-1"}}' + - name: task-name + value: for-loop-1 name: iteration-item-driver template: system-dag-driver - arguments: @@ -902,6 +912,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-1"},"inputs":{"parameters":{"pipelinechannel--loop_parameter":{"componentInputParameter":"loop_parameter"}}},"iteratorPolicy":{"parallelismLimit":2},"parameterIterator":{"itemInput":"pipelinechannel--loop_parameter-loop-item","items":{"inputParameter":"pipelinechannel--loop_parameter"}},"taskInfo":{"name":"for-loop-1"}}' + - name: task-name + value: for-loop-1 name: iteration-driver template: system-dag-driver - arguments: @@ -934,6 +946,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-10"},"dependentTasks":["list-dict-maker-1"],"inputs":{"parameters":{"pipelinechannel--list-dict-maker-1-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"list-dict-maker-1"}}}},"parameterIterator":{"itemInput":"pipelinechannel--list-dict-maker-1-Output-loop-item","items":{"inputParameter":"pipelinechannel--list-dict-maker-1-Output"}},"taskInfo":{"name":"for-loop-10"}}' + - name: task-name + value: for-loop-10 name: iteration-item-driver template: system-dag-driver - arguments: @@ -962,6 +976,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-10"},"dependentTasks":["list-dict-maker-1"],"inputs":{"parameters":{"pipelinechannel--list-dict-maker-1-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"list-dict-maker-1"}}}},"parameterIterator":{"itemInput":"pipelinechannel--list-dict-maker-1-Output-loop-item","items":{"inputParameter":"pipelinechannel--list-dict-maker-1-Output"}},"taskInfo":{"name":"for-loop-10"}}' + - name: task-name + value: for-loop-10 name: iteration-driver template: system-dag-driver - arguments: @@ -993,6 +1009,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-11"},"dependentTasks":["list-dict-maker-2"],"inputs":{"parameters":{"pipelinechannel--list-dict-maker-2-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"list-dict-maker-2"}}}},"parameterIterator":{"itemInput":"pipelinechannel--list-dict-maker-2-Output-loop-item","items":{"inputParameter":"pipelinechannel--list-dict-maker-2-Output"}},"taskInfo":{"name":"for-loop-11"}}' + - name: task-name + value: for-loop-11 name: iteration-item-driver template: system-dag-driver - arguments: @@ -1021,6 +1039,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-11"},"dependentTasks":["list-dict-maker-2"],"inputs":{"parameters":{"pipelinechannel--list-dict-maker-2-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"list-dict-maker-2"}}}},"parameterIterator":{"itemInput":"pipelinechannel--list-dict-maker-2-Output-loop-item","items":{"inputParameter":"pipelinechannel--list-dict-maker-2-Output"}},"taskInfo":{"name":"for-loop-11"}}' + - name: task-name + value: for-loop-11 name: iteration-driver template: system-dag-driver - arguments: @@ -1052,6 +1072,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-12"},"dependentTasks":["list-dict-maker-3"],"inputs":{"parameters":{"pipelinechannel--list-dict-maker-3-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"list-dict-maker-3"}}}},"parameterIterator":{"itemInput":"pipelinechannel--list-dict-maker-3-Output-loop-item","items":{"inputParameter":"pipelinechannel--list-dict-maker-3-Output"}},"taskInfo":{"name":"for-loop-12"}}' + - name: task-name + value: for-loop-12 name: iteration-item-driver template: system-dag-driver - arguments: @@ -1080,6 +1102,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-12"},"dependentTasks":["list-dict-maker-3"],"inputs":{"parameters":{"pipelinechannel--list-dict-maker-3-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"list-dict-maker-3"}}}},"parameterIterator":{"itemInput":"pipelinechannel--list-dict-maker-3-Output-loop-item","items":{"inputParameter":"pipelinechannel--list-dict-maker-3-Output"}},"taskInfo":{"name":"for-loop-12"}}' + - name: task-name + value: for-loop-12 name: iteration-driver template: system-dag-driver - arguments: @@ -1111,6 +1135,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-13"},"dependentTasks":["list-dict-maker-1-2"],"inputs":{"parameters":{"pipelinechannel--list-dict-maker-1-2-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"list-dict-maker-1-2"}}}},"parameterIterator":{"itemInput":"pipelinechannel--list-dict-maker-1-2-Output-loop-item","items":{"inputParameter":"pipelinechannel--list-dict-maker-1-2-Output"}},"taskInfo":{"name":"for-loop-13"}}' + - name: task-name + value: for-loop-13 name: iteration-item-driver template: system-dag-driver - arguments: @@ -1139,6 +1165,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-13"},"dependentTasks":["list-dict-maker-1-2"],"inputs":{"parameters":{"pipelinechannel--list-dict-maker-1-2-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"list-dict-maker-1-2"}}}},"parameterIterator":{"itemInput":"pipelinechannel--list-dict-maker-1-2-Output-loop-item","items":{"inputParameter":"pipelinechannel--list-dict-maker-1-2-Output"}},"taskInfo":{"name":"for-loop-13"}}' + - name: task-name + value: for-loop-13 name: iteration-driver template: system-dag-driver - arguments: @@ -1171,6 +1199,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-4"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-3","items":{"raw":"[{\"A_a\": \"1\", \"B_b\": \"2\"}, {\"A_a\": \"10\", \"B_b\": \"20\"}]"}},"taskInfo":{"name":"for-loop-4"}}' + - name: task-name + value: for-loop-4 name: iteration-item-driver template: system-dag-driver - arguments: @@ -1200,6 +1230,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-4"},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-3","items":{"raw":"[{\"A_a\": \"1\", \"B_b\": \"2\"}, {\"A_a\": \"10\", \"B_b\": \"20\"}]"}},"taskInfo":{"name":"for-loop-4"}}' + - name: task-name + value: for-loop-4 name: iteration-driver template: system-dag-driver - arguments: @@ -1232,6 +1264,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-8"},"iteratorPolicy":{"parallelismLimit":1},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-7","items":{"raw":"[{\"a\": 1, \"b\": 2}, {\"a\": 2, \"b\": 3}, {\"a\": 3, \"b\": 4}]"}},"taskInfo":{"name":"for-loop-8"}}' + - name: task-name + value: for-loop-8 name: iteration-item-driver template: system-dag-driver - arguments: @@ -1261,6 +1295,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-8"},"iteratorPolicy":{"parallelismLimit":1},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-7","items":{"raw":"[{\"a\": 1, \"b\": 2}, {\"a\": 2, \"b\": 3}, {\"a\": 3, \"b\": 4}]"}},"taskInfo":{"name":"for-loop-8"}}' + - name: task-name + value: for-loop-8 name: iteration-driver template: system-dag-driver - arguments: @@ -1293,6 +1329,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-9"},"dependentTasks":["list-dict-maker-0"],"inputs":{"parameters":{"pipelinechannel--list-dict-maker-0-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"list-dict-maker-0"}}}},"parameterIterator":{"itemInput":"pipelinechannel--list-dict-maker-0-Output-loop-item","items":{"inputParameter":"pipelinechannel--list-dict-maker-0-Output"}},"taskInfo":{"name":"for-loop-9"}}' + - name: task-name + value: for-loop-9 name: iteration-item-driver template: system-dag-driver - arguments: @@ -1321,6 +1359,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-9"},"dependentTasks":["list-dict-maker-0"],"inputs":{"parameters":{"pipelinechannel--list-dict-maker-0-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"list-dict-maker-0"}}}},"parameterIterator":{"itemInput":"pipelinechannel--list-dict-maker-0-Output-loop-item","items":{"inputParameter":"pipelinechannel--list-dict-maker-0-Output"}},"taskInfo":{"name":"for-loop-9"}}' + - name: task-name + value: for-loop-9 name: iteration-driver template: system-dag-driver - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_params_containing_format.yaml b/test_data/compiled-workflows/pipeline_with_params_containing_format.yaml index eb0e3b2db7c..9cfa82134cc 100644 --- a/test_data/compiled-workflows/pipeline_with_params_containing_format.yaml +++ b/test_data/compiled-workflows/pipeline_with_params_containing_format.yaml @@ -183,7 +183,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: @@ -329,6 +329,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"inputs":{"parameters":{"pipelinechannel--name":{"componentInputParameter":"name"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[\"1\", \"2\"]"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-item-driver template: system-dag-driver - arguments: @@ -358,6 +360,8 @@ spec: - name: task value: '{"componentRef":{"name":"comp-for-loop-2"},"inputs":{"parameters":{"pipelinechannel--name":{"componentInputParameter":"name"}}},"parameterIterator":{"itemInput":"pipelinechannel--loop-item-param-1","items":{"raw":"[\"1\", \"2\"]"}},"taskInfo":{"name":"for-loop-2"}}' + - name: task-name + value: for-loop-2 name: iteration-driver template: system-dag-driver - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_placeholders.yaml b/test_data/compiled-workflows/pipeline_with_placeholders.yaml index d4b020f95ca..6e7e89c45c3 100644 --- a/test_data/compiled-workflows/pipeline_with_placeholders.yaml +++ b/test_data/compiled-workflows/pipeline_with_placeholders.yaml @@ -170,7 +170,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_pod_metadata.yaml b/test_data/compiled-workflows/pipeline_with_pod_metadata.yaml index 0006d6536cc..70e1498a6af 100644 --- a/test_data/compiled-workflows/pipeline_with_pod_metadata.yaml +++ b/test_data/compiled-workflows/pipeline_with_pod_metadata.yaml @@ -285,7 +285,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: @@ -411,7 +411,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: @@ -572,7 +572,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: @@ -715,7 +715,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: @@ -848,7 +848,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_retry.yaml b/test_data/compiled-workflows/pipeline_with_retry.yaml index 7fb9adc9c1d..f5d51f8f703 100644 --- a/test_data/compiled-workflows/pipeline_with_retry.yaml +++ b/test_data/compiled-workflows/pipeline_with_retry.yaml @@ -179,7 +179,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_reused_component.yaml b/test_data/compiled-workflows/pipeline_with_reused_component.yaml index c077325071e..b5b85eca91e 100644 --- a/test_data/compiled-workflows/pipeline_with_reused_component.yaml +++ b/test_data/compiled-workflows/pipeline_with_reused_component.yaml @@ -163,7 +163,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml b/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml index de483de2be4..10163ffbcd4 100644 --- a/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml +++ b/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml @@ -182,7 +182,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_secret_as_volume.yaml b/test_data/compiled-workflows/pipeline_with_secret_as_volume.yaml index b1ba7fe8409..c571d987e8a 100644 --- a/test_data/compiled-workflows/pipeline_with_secret_as_volume.yaml +++ b/test_data/compiled-workflows/pipeline_with_secret_as_volume.yaml @@ -170,7 +170,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_string_machine_fields_pipeline_input.yaml b/test_data/compiled-workflows/pipeline_with_string_machine_fields_pipeline_input.yaml index b6a53429642..dd949a8af53 100644 --- a/test_data/compiled-workflows/pipeline_with_string_machine_fields_pipeline_input.yaml +++ b/test_data/compiled-workflows/pipeline_with_string_machine_fields_pipeline_input.yaml @@ -163,7 +163,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml b/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml index ffc87b9199c..4a58110eef5 100644 --- a/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml +++ b/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml @@ -72,7 +72,7 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef sum_numbers(a: int, b: int) -\u003e int:\n return a + b\n\n"],"image":"python:3.11","resources":{"accelerator":{"resourceCount":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}","resourceType":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"},"resourceCpuLimit":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}","resourceMemoryLimit":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}}' - name: components-root - value: '{"dag":{"tasks":{"accelerator-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-limit"},"taskInfo":{"name":"accelerator-limit"}},"accelerator-type":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-type"},"taskInfo":{"name":"accelerator-type"}},"cpu-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-cpu-limit"},"taskInfo":{"name":"cpu-limit"}},"memory-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-memory-limit"},"taskInfo":{"name":"memory-limit"}},"sum-numbers":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-limit","cpu-limit","memory-limit","accelerator-type"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}}}}' + value: '{"dag":{"tasks":{"accelerator-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-limit"},"taskInfo":{"name":"accelerator-limit"}},"accelerator-type":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-type"},"taskInfo":{"name":"accelerator-type"}},"cpu-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-cpu-limit"},"taskInfo":{"name":"cpu-limit"}},"memory-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-memory-limit"},"taskInfo":{"name":"memory-limit"}},"sum-numbers":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-type","memory-limit","accelerator-limit","cpu-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}}}}' entrypoint: entrypoint podMetadata: annotations: @@ -215,7 +215,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: @@ -357,15 +357,15 @@ spec: - name: component value: '{{workflow.parameters.components-49f9a898b718a077f30b7fd8c02d39767cff91ff0bbda4379daf866a91dbdb1b}}' - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-limit","cpu-limit","memory-limit","accelerator-type"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}' + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-type","memory-limit","accelerator-limit","cpu-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}' - name: container value: '{{workflow.parameters.implementations-49f9a898b718a077f30b7fd8c02d39767cff91ff0bbda4379daf866a91dbdb1b}}' - name: task-name value: sum-numbers - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - depends: accelerator-limit.Succeeded && cpu-limit.Succeeded && memory-limit.Succeeded - && accelerator-type.Succeeded + depends: accelerator-type.Succeeded && memory-limit.Succeeded && accelerator-limit.Succeeded + && cpu-limit.Succeeded name: sum-numbers-driver template: system-container-driver - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_submit_request.yaml b/test_data/compiled-workflows/pipeline_with_submit_request.yaml index c01c33d78ec..9f22537055d 100644 --- a/test_data/compiled-workflows/pipeline_with_submit_request.yaml +++ b/test_data/compiled-workflows/pipeline_with_submit_request.yaml @@ -174,7 +174,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_task_final_status.yaml b/test_data/compiled-workflows/pipeline_with_task_final_status.yaml index 33d57996017..2a3d98ef662 100644 --- a/test_data/compiled-workflows/pipeline_with_task_final_status.yaml +++ b/test_data/compiled-workflows/pipeline_with_task_final_status.yaml @@ -197,7 +197,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_task_final_status_yaml.yaml b/test_data/compiled-workflows/pipeline_with_task_final_status_yaml.yaml index 941f1ef2c8f..617a4b0a10c 100644 --- a/test_data/compiled-workflows/pipeline_with_task_final_status_yaml.yaml +++ b/test_data/compiled-workflows/pipeline_with_task_final_status_yaml.yaml @@ -162,7 +162,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_task_using_ignore_upstream_failure.yaml b/test_data/compiled-workflows/pipeline_with_task_using_ignore_upstream_failure.yaml index c645ffa99d4..fabbaf644a2 100644 --- a/test_data/compiled-workflows/pipeline_with_task_using_ignore_upstream_failure.yaml +++ b/test_data/compiled-workflows/pipeline_with_task_using_ignore_upstream_failure.yaml @@ -177,7 +177,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_utils.yaml b/test_data/compiled-workflows/pipeline_with_utils.yaml index b874a5268e1..229179e7651 100644 --- a/test_data/compiled-workflows/pipeline_with_utils.yaml +++ b/test_data/compiled-workflows/pipeline_with_utils.yaml @@ -167,7 +167,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_various_io_types.yaml b/test_data/compiled-workflows/pipeline_with_various_io_types.yaml index 1016d977ca4..27dce2d5bfd 100644 --- a/test_data/compiled-workflows/pipeline_with_various_io_types.yaml +++ b/test_data/compiled-workflows/pipeline_with_various_io_types.yaml @@ -158,7 +158,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_volume.yaml b/test_data/compiled-workflows/pipeline_with_volume.yaml index b0bb74e9f0f..3bbe1cce98e 100644 --- a/test_data/compiled-workflows/pipeline_with_volume.yaml +++ b/test_data/compiled-workflows/pipeline_with_volume.yaml @@ -211,7 +211,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_volume_no_cache.yaml b/test_data/compiled-workflows/pipeline_with_volume_no_cache.yaml index e3e0f571864..fc9492b932c 100644 --- a/test_data/compiled-workflows/pipeline_with_volume_no_cache.yaml +++ b/test_data/compiled-workflows/pipeline_with_volume_no_cache.yaml @@ -211,7 +211,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pipeline_with_workspace.yaml b/test_data/compiled-workflows/pipeline_with_workspace.yaml index 6f6cdfe067f..0c12a46ae0f 100644 --- a/test_data/compiled-workflows/pipeline_with_workspace.yaml +++ b/test_data/compiled-workflows/pipeline_with_workspace.yaml @@ -187,7 +187,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/placeholder_with_if_placeholder_none_input_value.yaml b/test_data/compiled-workflows/placeholder_with_if_placeholder_none_input_value.yaml index 3cf07a1a0ed..f0df5106f0a 100644 --- a/test_data/compiled-workflows/placeholder_with_if_placeholder_none_input_value.yaml +++ b/test_data/compiled-workflows/placeholder_with_if_placeholder_none_input_value.yaml @@ -157,7 +157,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/preprocess.yaml b/test_data/compiled-workflows/preprocess.yaml index 8501a6d591f..833d9ece30b 100644 --- a/test_data/compiled-workflows/preprocess.yaml +++ b/test_data/compiled-workflows/preprocess.yaml @@ -185,7 +185,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/producer_consumer_param_pipeline.yaml b/test_data/compiled-workflows/producer_consumer_param_pipeline.yaml index 90ee21a7cc3..6ebcfb8baa6 100644 --- a/test_data/compiled-workflows/producer_consumer_param_pipeline.yaml +++ b/test_data/compiled-workflows/producer_consumer_param_pipeline.yaml @@ -161,7 +161,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pvc_mount.yaml b/test_data/compiled-workflows/pvc_mount.yaml index c4f83b53063..63438a876da 100644 --- a/test_data/compiled-workflows/pvc_mount.yaml +++ b/test_data/compiled-workflows/pvc_mount.yaml @@ -181,7 +181,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml b/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml index 6db48d8a78e..77ae4745ea9 100644 --- a/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml +++ b/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml @@ -243,7 +243,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pythonic_artifacts_test_pipeline.yaml b/test_data/compiled-workflows/pythonic_artifacts_test_pipeline.yaml index 79c2e43a913..160729f5014 100644 --- a/test_data/compiled-workflows/pythonic_artifacts_test_pipeline.yaml +++ b/test_data/compiled-workflows/pythonic_artifacts_test_pipeline.yaml @@ -182,7 +182,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/pythonic_artifacts_with_list_of_artifacts.yaml b/test_data/compiled-workflows/pythonic_artifacts_with_list_of_artifacts.yaml index dcbbef39511..7baaa5c1106 100644 --- a/test_data/compiled-workflows/pythonic_artifacts_with_list_of_artifacts.yaml +++ b/test_data/compiled-workflows/pythonic_artifacts_with_list_of_artifacts.yaml @@ -183,7 +183,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: @@ -327,6 +327,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-1"},"inputs":{"parameters":{"pipelinechannel--texts":{"componentInputParameter":"texts"}}},"parameterIterator":{"itemInput":"pipelinechannel--texts-loop-item","items":{"inputParameter":"pipelinechannel--texts"}},"taskInfo":{"name":"for-loop-1"}}' + - name: task-name + value: for-loop-1 name: iteration-item-driver template: system-dag-driver - arguments: @@ -355,6 +357,8 @@ spec: value: '{{inputs.parameters.parent-dag-id}}' - name: task value: '{"componentRef":{"name":"comp-for-loop-1"},"inputs":{"parameters":{"pipelinechannel--texts":{"componentInputParameter":"texts"}}},"parameterIterator":{"itemInput":"pipelinechannel--texts-loop-item","items":{"inputParameter":"pipelinechannel--texts"}},"taskInfo":{"name":"for-loop-1"}}' + - name: task-name + value: for-loop-1 name: iteration-driver template: system-dag-driver - arguments: diff --git a/test_data/compiled-workflows/pythonic_artifacts_with_multiple_returns.yaml b/test_data/compiled-workflows/pythonic_artifacts_with_multiple_returns.yaml index 032e1fa41e2..961dffa401c 100644 --- a/test_data/compiled-workflows/pythonic_artifacts_with_multiple_returns.yaml +++ b/test_data/compiled-workflows/pythonic_artifacts_with_multiple_returns.yaml @@ -188,7 +188,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/ray_integration_compiled.yaml b/test_data/compiled-workflows/ray_integration_compiled.yaml index a1bf92c1da7..f223802e41e 100644 --- a/test_data/compiled-workflows/ray_integration_compiled.yaml +++ b/test_data/compiled-workflows/ray_integration_compiled.yaml @@ -228,7 +228,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/run_as_user_cache_disabled.yaml b/test_data/compiled-workflows/run_as_user_cache_disabled.yaml index 0509104b2ba..710f2d1af0e 100644 --- a/test_data/compiled-workflows/run_as_user_cache_disabled.yaml +++ b/test_data/compiled-workflows/run_as_user_cache_disabled.yaml @@ -158,7 +158,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/run_as_user_cache_enabled.yaml b/test_data/compiled-workflows/run_as_user_cache_enabled.yaml index 0509104b2ba..710f2d1af0e 100644 --- a/test_data/compiled-workflows/run_as_user_cache_enabled.yaml +++ b/test_data/compiled-workflows/run_as_user_cache_enabled.yaml @@ -158,7 +158,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/sequential_v1.yaml b/test_data/compiled-workflows/sequential_v1.yaml index 2c76d1e86b4..121f9e3ad6e 100644 --- a/test_data/compiled-workflows/sequential_v1.yaml +++ b/test_data/compiled-workflows/sequential_v1.yaml @@ -154,7 +154,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/sequential_v2.yaml b/test_data/compiled-workflows/sequential_v2.yaml index 20e6a0b2cc1..6f5db7ce06f 100644 --- a/test_data/compiled-workflows/sequential_v2.yaml +++ b/test_data/compiled-workflows/sequential_v2.yaml @@ -158,7 +158,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/take_nap_compiled.yaml b/test_data/compiled-workflows/take_nap_compiled.yaml index 2e69c97f814..5dd45c9c84f 100644 --- a/test_data/compiled-workflows/take_nap_compiled.yaml +++ b/test_data/compiled-workflows/take_nap_compiled.yaml @@ -179,7 +179,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/take_nap_pipeline_root_compiled.yaml b/test_data/compiled-workflows/take_nap_pipeline_root_compiled.yaml index 2e69c97f814..5dd45c9c84f 100644 --- a/test_data/compiled-workflows/take_nap_pipeline_root_compiled.yaml +++ b/test_data/compiled-workflows/take_nap_pipeline_root_compiled.yaml @@ -179,7 +179,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/two_step_pipeline.yaml b/test_data/compiled-workflows/two_step_pipeline.yaml index 94aef95181d..94b3ca8e914 100644 --- a/test_data/compiled-workflows/two_step_pipeline.yaml +++ b/test_data/compiled-workflows/two_step_pipeline.yaml @@ -159,7 +159,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/two_step_pipeline_containerized.yaml b/test_data/compiled-workflows/two_step_pipeline_containerized.yaml index 9e2b9d852d7..a6739e86b03 100644 --- a/test_data/compiled-workflows/two_step_pipeline_containerized.yaml +++ b/test_data/compiled-workflows/two_step_pipeline_containerized.yaml @@ -160,7 +160,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/upload_download_compiled.yaml b/test_data/compiled-workflows/upload_download_compiled.yaml index 803f5a3517a..67bc3a8b150 100644 --- a/test_data/compiled-workflows/upload_download_compiled.yaml +++ b/test_data/compiled-workflows/upload_download_compiled.yaml @@ -218,7 +218,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: diff --git a/test_data/compiled-workflows/xgboost_sample_pipeline.yaml b/test_data/compiled-workflows/xgboost_sample_pipeline.yaml index 302700a6945..88199c14e89 100644 --- a/test_data/compiled-workflows/xgboost_sample_pipeline.yaml +++ b/test_data/compiled-workflows/xgboost_sample_pipeline.yaml @@ -422,7 +422,7 @@ spec: resources: limits: cpu: 500m - memory: 256Mi + memory: 128Mi requests: cpu: 100m securityContext: From ca7eb1099454655f7ac30bee9575529f7c9b2ec2 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Sun, 8 Mar 2026 17:53:52 +0300 Subject: [PATCH 32/51] - store driver logs into artifact store Signed-off-by: arpechenin --- .../manifests/base/driver-plugin-cm-path.yaml | 16 ++ .../tls-enabled/driver-plugin-cm-path.yaml | 14 ++ backend/src/common/util/context_logger.go | 68 ++++++++ backend/src/driver/main.go | 2 - backend/src/driver/rpc_handler.go | 146 ++++++++++++++++-- backend/src/v2/driver/container.go | 25 ++- backend/src/v2/driver/dag.go | 34 ++-- backend/src/v2/driver/driver.go | 12 +- backend/src/v2/driver/k8s.go | 52 ++++--- backend/src/v2/driver/root_dag.go | 37 +++-- .../pipelines-profile-controller/sync.py | 88 +++++------ .../ml-pipeline-driver-plugin-cm.yaml | 16 ++ .../base/pipeline/pipeline-runner-role.yaml | 26 ++++ .../patches/ml-pipeline-driver-plugin-cm.yaml | 14 ++ .../env/dev/driver-plugin-cm-path.yaml | 17 +- 15 files changed, 442 insertions(+), 125 deletions(-) create mode 100644 backend/src/common/util/context_logger.go diff --git a/.github/resources/manifests/base/driver-plugin-cm-path.yaml b/.github/resources/manifests/base/driver-plugin-cm-path.yaml index 40bcc51d190..08f446793e7 100644 --- a/.github/resources/manifests/base/driver-plugin-cm-path.yaml +++ b/.github/resources/manifests/base/driver-plugin-cm-path.yaml @@ -7,6 +7,17 @@ data: name: driver-plugin image: kind-registry:5000/driver:latest imagePullPolicy: IfNotPresent + env: + - name: LOG_ACCESS_KEY + valueFrom: + secretKeyRef: + name: mlpipeline-minio-artifact + key: accesskey + - name: LOG_SECRET_KEY + valueFrom: + secretKeyRef: + name: mlpipeline-minio-artifact + key: secretkey ports: - containerPort: 8080 resources: @@ -25,4 +36,9 @@ data: - ALL seccompProfile: type: RuntimeDefault + volumeMounts: + - name: var-run-argo + mountPath: /kfp/log + readOnly: false + diff --git a/.github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml b/.github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml index e3ea264b4c9..1f88b869662 100644 --- a/.github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml +++ b/.github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml @@ -9,6 +9,17 @@ data: imagePullPolicy: IfNotPresent ports: - containerPort: 8080 + env: + - name: LOG_ACCESS_KEY + valueFrom: + secretKeyRef: + name: mlpipeline-minio-artifact + key: accesskey + - name: LOG_SECRET_KEY + valueFrom: + secretKeyRef: + name: mlpipeline-minio-artifact + key: secretkey resources: requests: cpu: 100m @@ -29,4 +40,7 @@ data: - name: argo-workflows-agent-ca-certificates mountPath: /kfp/certs readOnly: true + - name: var-run-argo + mountPath: /kfp/log + readOnly: false diff --git a/backend/src/common/util/context_logger.go b/backend/src/common/util/context_logger.go new file mode 100644 index 00000000000..2c41a25ec76 --- /dev/null +++ b/backend/src/common/util/context_logger.go @@ -0,0 +1,68 @@ +package util + +import ( + "context" + "fmt" + "io" + "os" + + "github.com/sirupsen/logrus" +) + +type CtxKey string + +const ( + contextLoggerKey CtxKey = "driver_log_key" +) + +func newFileLogger(logFile string) (*logrus.Logger, io.Closer, error) { + f, err := os.Create(logFile) + if err != nil { + return nil, nil, err + } + + logger := logrus.New() + logger.Out = io.MultiWriter(os.Stdout, f) + logger.Formatter = &logrus.TextFormatter{} + return logger, f, nil +} + +func WithLogger(ctx context.Context, logFile string) (context.Context, io.Closer, error) { + if ctx == nil { + return nil, nil, fmt.Errorf( + "error during creation of the logger for logId: %v. ctx can not be nil", + logFile, + ) + } + + if GetLoggerFrom(ctx) != nil { + return nil, nil, fmt.Errorf("logger already exists in context") + } + + logger, f, err := newFileLogger(logFile) + if err != nil { + return nil, nil, fmt.Errorf( + "error during creation of the logger for logId: %v details: %w", + logFile, + err, + ) + } + + ctx = context.WithValue(ctx, contextLoggerKey, logger) + + return ctx, f, nil +} + +func GetLoggerFrom(ctx context.Context) *logrus.Logger { + v := ctx.Value(contextLoggerKey) + if v == nil { + return nil + } + + logger, ok := v.(*logrus.Logger) + if !ok { + return nil + } + + return logger +} diff --git a/backend/src/driver/main.go b/backend/src/driver/main.go index 70590b2ea62..877b5e06532 100644 --- a/backend/src/driver/main.go +++ b/backend/src/driver/main.go @@ -75,7 +75,6 @@ func init() { func parseExecConfigJSON(k8sExecConfigJSON *string) (*kubernetesplatform.KubernetesExecutorConfig, error) { var k8sExecCfg *kubernetesplatform.KubernetesExecutorConfig if *k8sExecConfigJSON != "" { - glog.Infof("input kubernetesConfig:%s\n", prettyPrint(*k8sExecConfigJSON)) k8sExecCfg = &kubernetesplatform.KubernetesExecutorConfig{} if err := util.UnmarshalString(*k8sExecConfigJSON, k8sExecCfg); err != nil { return nil, fmt.Errorf("failed to unmarshal Kubernetes config, error: %w\nKubernetesConfig: %v", err, k8sExecConfigJSON) @@ -166,6 +165,5 @@ func writeFile(path string, data []byte) (err error) { } func newMlmdClient(mlmdServerAddress string, mlmdServerPort string, tlsCfg *tls.Config) (*metadata.Client, error) { - glog.Infof("mlmd server address: %s:%s\n", mlmdServerAddress, mlmdServerPort) return metadata.NewClient(mlmdServerAddress, mlmdServerPort, tlsCfg) } diff --git a/backend/src/driver/rpc_handler.go b/backend/src/driver/rpc_handler.go index de50f2e7a07..4b39ec48a0b 100644 --- a/backend/src/driver/rpc_handler.go +++ b/backend/src/driver/rpc_handler.go @@ -26,6 +26,7 @@ import ( "strings" "github.com/golang/glog" + "github.com/google/uuid" "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" "github.com/kubeflow/pipelines/backend/src/apiserver/config/proxy" "github.com/kubeflow/pipelines/backend/src/common/util" @@ -33,9 +34,23 @@ import ( "github.com/kubeflow/pipelines/backend/src/v2/cacheutils" "github.com/kubeflow/pipelines/backend/src/v2/config" "github.com/kubeflow/pipelines/backend/src/v2/driver" + "github.com/kubeflow/pipelines/backend/src/v2/metadata" + "github.com/kubeflow/pipelines/backend/src/v2/objectstore" "google.golang.org/protobuf/encoding/protojson" + "k8s.io/client-go/kubernetes" ) +type driverLogArtifactContext struct { + Execution *driver.Execution + Task string + LocalPath string + OutputPathPrefix string + Namespace string + PipelineRoot string + StoreSessionInfo string + LogId string +} + func ExecutePlugin(w http.ResponseWriter, r *http.Request) { defer func(Body io.ReadCloser) { err := Body.Close() @@ -133,7 +148,54 @@ func drive(args api.DriverPluginArgs) (execution *driver.Execution, err error) { err = fmt.Errorf("KFP driver: %w", err) } }() - ctx := context.Background() + var ( + pipelineRoot string + storeSessionInfo string + namespace string + outputPathPrefix string + ) + var pipeline *metadata.Pipeline + logId := fmt.Sprintf("%v-%v-%v", args.IterationIndex, args.Type, args.TaskName) + logDir := "/kfp/log" + logFile := fmt.Sprintf("%s/%s.log", logDir, logId) + ctx, f, err := util.WithLogger(context.Background(), logFile) + if err != nil { + return nil, fmt.Errorf("failed to create driver logger: %v", err) + } + defer func() { + err = os.Remove(logFile) + if err != nil { + glog.Errorf("Failed to remove processed log file: %v", err) + } + }() + defer func() { + if pipelineRoot != "" { + logContext := &driverLogArtifactContext{ + Execution: execution, + Task: args.TaskName, + LocalPath: logFile, + LogId: logId, + Namespace: namespace, + PipelineRoot: pipelineRoot, + StoreSessionInfo: storeSessionInfo, + OutputPathPrefix: outputPathPrefix, + } + uploadErr := uploadDriverLogArtifact(ctx, logContext) + if uploadErr != nil { + glog.Errorf("Failed to upload driver-logs artifact: %v", uploadErr) + } + } + }() + defer func() { + if f != nil { + err = f.Close() + if err != nil { + glog.Errorf("Failed to close file: %v", err) + } + } + }() + + log := util.GetLoggerFrom(ctx) // Support reading component spec from a file if value starts with @ // This bypasses exec() argument size limits for large workflows @@ -144,19 +206,19 @@ func drive(args api.DriverPluginArgs) (execution *driver.Execution, err error) { return nil, fmt.Errorf("failed to read component spec from file %s: %w", filePath, err) } args.Component = string(data) - glog.Infof("Read component spec from file: %s (%d bytes)", filePath, len(data)) + log.Infof("Read component spec from file: %s (%d bytes)", filePath, len(data)) } proxy.InitializeConfig(args.HTTPProxy, args.HTTPSProxy, args.NoProxy) - glog.Infof("input ComponentSpec:%s\n", prettyPrint(args.Component)) + log.Infof("input ComponentSpec:%s\n", prettyPrint(args.Component)) componentSpec := &pipelinespec.ComponentSpec{} if err := util.UnmarshalString(args.Component, componentSpec); err != nil { return nil, fmt.Errorf("failed to unmarshal component spec, error: %w\ncomponentSpec: %v", err, prettyPrint(args.Component)) } var taskSpec *pipelinespec.PipelineTaskSpec if args.Task != "" { - glog.Infof("input TaskSpec:%s\n", prettyPrint(args.Task)) + log.Infof("input TaskSpec:%s\n", prettyPrint(args.Task)) taskSpec = &pipelinespec.PipelineTaskSpec{} if err := util.UnmarshalString(args.Task, taskSpec); err != nil { return nil, fmt.Errorf("failed to unmarshal task spec, error: %w\ntask: %v", err, args.Task) @@ -165,24 +227,27 @@ func drive(args api.DriverPluginArgs) (execution *driver.Execution, err error) { containerSpec := &pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec{} if args.Container != "" { - glog.Infof("input ContainerSpec:%s\n", prettyPrint(args.Container)) + log.Infof("input ContainerSpec:%s\n", prettyPrint(args.Container)) if err := util.UnmarshalString(args.Container, containerSpec); err != nil { return nil, fmt.Errorf("failed to unmarshal container spec, error: %w\ncontainerSpec: %v", err, args.Container) } } var runtimeConfig *pipelinespec.PipelineJob_RuntimeConfig if args.RuntimeConfig != "" { - glog.Infof("input RuntimeConfig:%s\n", prettyPrint(args.RuntimeConfig)) + log.Infof("input RuntimeConfig:%s\n", prettyPrint(args.RuntimeConfig)) runtimeConfig = &pipelinespec.PipelineJob_RuntimeConfig{} if err := util.UnmarshalString(args.RuntimeConfig, runtimeConfig); err != nil { return nil, fmt.Errorf("failed to unmarshal runtime config, error: %w\nruntimeConfig: %v", err, args.RuntimeConfig) } } + if args.KubernetesConfig != "" { + log.Infof("input kubernetesConfig:%s\n", prettyPrint(args.KubernetesConfig)) + } k8sExecCfg, err := parseExecConfigJSON(&args.KubernetesConfig) if err != nil { return nil, err } - namespace, err := config.InPodNamespace() + namespace, err = config.InPodNamespace() if err != nil { return nil, err } @@ -238,9 +303,20 @@ func drive(args api.DriverPluginArgs) (execution *driver.Execution, err error) { switch args.Type { case RootDag: options.RuntimeConfig = runtimeConfig - execution, driverErr = driver.RootDAG(ctx, options, client) + execution, pipeline, driverErr = driver.RootDAG(ctx, options, client) + if driverErr != nil { + return nil, err + } + pipelineRoot = pipeline.GetPipelineRoot() + storeSessionInfo = pipeline.GetStoreSessionInfo() case DAG: - execution, driverErr = driver.DAG(ctx, options, client) + pipeline, driverErr = client.GetPipeline(ctx, options.PipelineName, options.RunID, "", "", "", "") + if driverErr != nil { + return nil, driverErr + } + pipelineRoot = pipeline.GetPipelineRoot() + storeSessionInfo = pipeline.GetStoreSessionInfo() + execution, driverErr = driver.DAG(ctx, pipeline, options, client) case CONTAINER: options.Container = containerSpec options.KubernetesExecutorConfig = k8sExecCfg @@ -256,11 +332,19 @@ func drive(args api.DriverPluginArgs) (execution *driver.Execution, err error) { options.DefaultRunAsNonRoot = &v } } - execution, driverErr = driver.Container(ctx, options, client, cacheClient) + pipeline, driverErr = client.GetPipeline(ctx, options.PipelineName, options.RunID, "", "", "", "") + if driverErr != nil { + return nil, driverErr + } + pipelineRoot = pipeline.GetPipelineRoot() + storeSessionInfo = pipeline.GetStoreSessionInfo() + outputPathPrefix = uuid.NewString() + execution, driverErr = driver.Container(ctx, pipeline, options, client, cacheClient, outputPathPrefix) default: err = fmt.Errorf("unknown driverType %s", args.Type) } if driverErr != nil { + log.Errorf("driver execution failed with error: %v", driverErr) if execution == nil { return nil, driverErr } @@ -275,6 +359,44 @@ func drive(args api.DriverPluginArgs) (execution *driver.Execution, err error) { return execution, nil } +func uploadDriverLogArtifact(ctx context.Context, logContext *driverLogArtifactContext) error { + if logContext == nil { + return fmt.Errorf("logContext is nil") + } + if logContext.PipelineRoot != "" { + restConfig, err := util.GetKubernetesConfig() + if err != nil { + return fmt.Errorf("failed to get kubernetes config: %v", err) + } + k8sClient, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("failed to initialize kubernetes client set: %w", err) + } + session, err := objectstore.GetSessionInfoFromString(logContext.StoreSessionInfo) + if err != nil { + return fmt.Errorf("failed to get session info from store: %v", err) + } + bucketConfig, err := objectstore.ParseBucketConfig(logContext.PipelineRoot, session) + if err != nil { + return fmt.Errorf("failed to parse bucket config: %v", err) + } + bucket, err := objectstore.OpenBucket(ctx, k8sClient, logContext.Namespace, bucketConfig) + if err != nil { + return fmt.Errorf("failed to open bucket: %v", err) + } + key := fmt.Sprintf("driver/%s-logs", logContext.LogId) + if logContext.Execution != nil && logContext.OutputPathPrefix != "" { + key = fmt.Sprintf("%s/%s/driver-logs", logContext.Task, logContext.OutputPathPrefix) + } + glog.Infof("Uploading log key: %s ...", key) + err = objectstore.UploadBlob(ctx, bucket, logContext.LocalPath, key) + if err != nil { + return fmt.Errorf("failed to upload log: %v", err) + } + } + return nil +} + func validate(args api.DriverPluginArgs) error { switch { case args.Type == "": @@ -338,3 +460,7 @@ func WriteJSONResponse(w http.ResponseWriter, payload api.DriverResponse) { http.Error(w, "failed to encode response", http.StatusInternalServerError) } } + +func putLog() { + +} diff --git a/backend/src/v2/driver/container.go b/backend/src/v2/driver/container.go index 7a39a38cef4..1383a803b5e 100644 --- a/backend/src/v2/driver/container.go +++ b/backend/src/v2/driver/container.go @@ -20,9 +20,8 @@ import ( "fmt" "strconv" - "github.com/golang/glog" - "github.com/google/uuid" "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" + "github.com/kubeflow/pipelines/backend/src/common/util" "github.com/kubeflow/pipelines/backend/src/v2/cacheutils" "github.com/kubeflow/pipelines/backend/src/v2/expression" "github.com/kubeflow/pipelines/backend/src/v2/metadata" @@ -43,14 +42,18 @@ func validateContainer(opts Options) (err error) { return validateNonRoot(opts) } -func Container(ctx context.Context, opts Options, mlmd *metadata.Client, cacheClient cacheutils.Client) (execution *Execution, err error) { +func Container(ctx context.Context, pipeline *metadata.Pipeline, opts Options, mlmd *metadata.Client, cacheClient cacheutils.Client, outputPathPrefix string) (execution *Execution, err error) { defer func() { if err != nil { err = fmt.Errorf("driver.Container(%s) failed: %w", opts.info(), err) } }() b, _ := json.Marshal(opts) - glog.V(4).Info("Container opts: ", string(b)) + log := util.GetLoggerFrom(ctx) + if log == nil { + return nil, fmt.Errorf("failed to get logger for container: %s", string(b)) + } + log.Trace("Container opts: ", string(b)) err = validateContainer(opts) if err != nil { return nil, err @@ -62,15 +65,11 @@ func Container(ctx context.Context, opts Options, mlmd *metadata.Client, cacheCl } // TODO(Bobgy): there's no need to pass any parameters, because pipeline // and pipeline run context have been created by root DAG driver. - pipeline, err := mlmd.GetPipeline(ctx, opts.PipelineName, opts.RunID, "", "", "", "") - if err != nil { - return nil, err - } dag, err := mlmd.GetDAG(ctx, opts.DAGExecutionID) if err != nil { return nil, err } - glog.Infof("parent DAG: %+v", dag.Execution) + log.Infof("parent DAG: %+v", dag.Execution) expr, err := expression.New() if err != nil { return nil, err @@ -116,7 +115,7 @@ func Container(ctx context.Context, opts Options, mlmd *metadata.Client, cacheCl pipeline.GetPipelineRoot(), opts.TaskName, opts.Component.GetOutputDefinitions(), - uuid.NewString(), + outputPathPrefix, opts.PublishLogs, ) } @@ -182,7 +181,7 @@ func Container(ctx context.Context, opts Options, mlmd *metadata.Client, cacheCl if err != nil { return execution, err } - glog.Infof("Created execution: %s", createdExecution) + log.Infof("Created execution: %s", createdExecution) execution.ID = createdExecution.GetID() if !execution.WillTrigger() { return execution, nil @@ -206,12 +205,12 @@ func Container(ctx context.Context, opts Options, mlmd *metadata.Client, cacheCl if err := mlmd.PublishExecution(ctx, createdExecution, executorOutput.GetParameterValues(), outputArtifacts, pb.Execution_CACHED); err != nil { return execution, fmt.Errorf("failed to publish cached execution: %w", err) } - glog.Infof("Use cache for task %s", opts.Task.GetTaskInfo().GetName()) + log.Infof("Use cache for task %s", opts.Task.GetTaskInfo().GetName()) *execution.Cached = true return execution, nil } } else { - glog.Info("Cache disabled globally at the server level.") + log.Info("Cache disabled globally at the server level.") } taskConfig := &TaskConfig{} diff --git a/backend/src/v2/driver/dag.go b/backend/src/v2/driver/dag.go index 362fac66f7f..d494610c872 100644 --- a/backend/src/v2/driver/dag.go +++ b/backend/src/v2/driver/dag.go @@ -19,8 +19,8 @@ import ( "encoding/json" "fmt" - "github.com/golang/glog" "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" + "github.com/kubeflow/pipelines/backend/src/common/util" "github.com/kubeflow/pipelines/backend/src/v2/expression" "github.com/kubeflow/pipelines/backend/src/v2/metadata" "google.golang.org/protobuf/types/known/structpb" @@ -38,14 +38,18 @@ func validateDAG(opts Options) (err error) { return validateNonRoot(opts) } -func DAG(ctx context.Context, opts Options, mlmd *metadata.Client) (execution *Execution, err error) { +func DAG(ctx context.Context, pipeline *metadata.Pipeline, opts Options, mlmd *metadata.Client) (execution *Execution, err error) { defer func() { if err != nil { err = fmt.Errorf("driver.DAG(%s) failed: %w", opts.info(), err) } }() b, _ := json.Marshal(opts) - glog.V(4).Info("DAG opts: ", string(b)) + log := util.GetLoggerFrom(ctx) + if log == nil { + return nil, fmt.Errorf("failed to get logger for DAG driver options: %s", string(b)) + } + log.Trace("DAG opts: ", string(b)) err = validateDAG(opts) if err != nil { return nil, err @@ -55,17 +59,11 @@ func DAG(ctx context.Context, opts Options, mlmd *metadata.Client) (execution *E index := opts.IterationIndex iterationIndex = &index } - // TODO(Bobgy): there's no need to pass any parameters, because pipeline - // and pipeline run context have been created by root DAG driver. - pipeline, err := mlmd.GetPipeline(ctx, opts.PipelineName, opts.RunID, "", "", "", "") - if err != nil { - return nil, err - } dag, err := mlmd.GetDAG(ctx, opts.DAGExecutionID) if err != nil { return nil, err } - glog.Infof("parent DAG: %+v", dag.Execution) + log.Infof("parent DAG: %+v", dag.Execution) expr, err := expression.New() if err != nil { return nil, err @@ -77,7 +75,7 @@ func DAG(ctx context.Context, opts Options, mlmd *metadata.Client) (execution *E executorInput := &pipelinespec.ExecutorInput{ Inputs: inputs, } - glog.Infof("executorInput value: %+v", executorInput) + log.Infof("executorInput value: %+v", executorInput) execution = &Execution{ExecutorInput: executorInput} condition := opts.Task.GetTriggerPolicy().GetCondition() if condition != "" { @@ -109,15 +107,15 @@ func DAG(ctx context.Context, opts Options, mlmd *metadata.Client) (execution *E // Handle writing output parameters to MLMD. ecfg.OutputParameters = opts.Component.GetDag().GetOutputs().GetParameters() - glog.V(4).Info("outputParameters: ", ecfg.OutputParameters) + log.Trace("outputParameters: ", ecfg.OutputParameters) // Handle writing output artifacts to MLMD. ecfg.OutputArtifacts = opts.Component.GetDag().GetOutputs().GetArtifacts() - glog.V(4).Info("outputArtifacts: ", ecfg.OutputArtifacts) + log.Trace("outputArtifacts: ", ecfg.OutputArtifacts) totalDagTasks := len(opts.Component.GetDag().GetTasks()) ecfg.TotalDagTasks = &totalDagTasks - glog.V(4).Info("totalDagTasks: ", *ecfg.TotalDagTasks) + log.Trace("totalDagTasks: ", *ecfg.TotalDagTasks) if opts.Task.GetArtifactIterator() != nil { return execution, fmt.Errorf("ArtifactIterator is not implemented") @@ -164,17 +162,17 @@ func DAG(ctx context.Context, opts Options, mlmd *metadata.Client) (execution *E execution.IterationCount = &count } - glog.V(4).Info("pipeline: ", pipeline) + log.Trace("pipeline: ", pipeline) b, _ = json.Marshal(*ecfg) - glog.V(4).Info("ecfg: ", string(b)) - glog.V(4).Infof("dag: %v", dag) + log.Trace("ecfg: ", string(b)) + log.Trace("dag: %v", dag) // TODO(Bobgy): change execution state to pending, because this is driver, execution hasn't started. createdExecution, err := mlmd.CreateExecution(ctx, pipeline, ecfg) if err != nil { return execution, err } - glog.Infof("Created execution: %s", createdExecution) + log.Infof("Created execution: %s", createdExecution) execution.ID = createdExecution.GetID() return execution, nil } diff --git a/backend/src/v2/driver/driver.go b/backend/src/v2/driver/driver.go index 27b21845e2f..ab12434b9df 100644 --- a/backend/src/v2/driver/driver.go +++ b/backend/src/v2/driver/driver.go @@ -696,7 +696,7 @@ func provisionOutputs( pipelineRoot, taskName string, outputsSpec *pipelinespec.ComponentOutputsSpec, - outputURISalt string, + prefix string, publishOutput string, ) *pipelinespec.ExecutorInput_Outputs { outputs := &pipelinespec.ExecutorInput_Outputs{ @@ -719,6 +719,14 @@ func provisionOutputs( }, }, } + artifacts["driver-logs"] = &pipelinespec.ComponentOutputsSpec_ArtifactSpec{ + ArtifactType: &pipelinespec.ArtifactTypeSchema{ + Kind: &pipelinespec.ArtifactTypeSchema_SchemaTitle{ + SchemaTitle: "system.Artifact", + }, + }, + } + } // Compute a task-root remote URI that will serve as the base for all @@ -726,7 +734,7 @@ func provisionOutputs( // artifacts (dsl.get_uri) by allowing the SDK to infer the task root from // the executor output file's directory (set below) and convert it back to // a remote URI at runtime. - taskRootRemote := metadata.GenerateOutputURI(pipelineRoot, []string{taskName, outputURISalt}, false) + taskRootRemote := metadata.GenerateOutputURI(pipelineRoot, []string{taskName, prefix}, false) // Set per-artifact output URIs under the task root. for name, artifact := range artifacts { diff --git a/backend/src/v2/driver/k8s.go b/backend/src/v2/driver/k8s.go index 845037dc27c..ae27d990f30 100644 --- a/backend/src/v2/driver/k8s.go +++ b/backend/src/v2/driver/k8s.go @@ -21,7 +21,6 @@ import ( "fmt" "time" - "github.com/golang/glog" "github.com/google/uuid" "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" "github.com/kubeflow/pipelines/backend/src/common/util" @@ -127,6 +126,10 @@ func extendPodSpecPatch( kubernetesExecutorConfig := opts.KubernetesExecutorConfig setOnTaskConfig, setOnPod := getTaskConfigOptions(opts.Component) + log := util.GetLoggerFrom(ctx) + if log == nil { + return fmt.Errorf("cannot get log from context for extendPodSpecPatch") + } // Always set setOnTaskConfig to an empty map if taskConfig is nil to avoid nil pointer dereference. if taskConfig == nil { @@ -210,7 +213,7 @@ func extendPodSpecPatch( if tolerations := kubernetesExecutorConfig.GetTolerations(); tolerations != nil { var k8sTolerations []k8score.Toleration - glog.Infof("Tolerations passed: %+v", tolerations) + log.Infof("Tolerations passed: %+v", tolerations) for _, toleration := range tolerations { if toleration != nil { @@ -244,7 +247,7 @@ func extendPodSpecPatch( } k8sTolerations = append(k8sTolerations, singleToleration) } else { - glog.V(4).Info("encountered empty tolerations struct, ignoring.") + log.Trace("encountered empty tolerations struct, ignoring.") } } else if isListToleration { listVal := resolvedParam.GetListValue() @@ -259,7 +262,7 @@ func extendPodSpecPatch( } k8sTolerations = append(k8sTolerations, k8sTolerationsList...) } else { - glog.V(4).Info("encountered empty tolerations list, ignoring.") + log.Trace("encountered empty tolerations list, ignoring.") } } else { return fmt.Errorf("encountered unexpected toleration proto value, must be either struct or list type") @@ -608,7 +611,7 @@ func extendPodSpecPatch( if nodeAffinityTerm.GetNodeAffinityJson() == nil && len(nodeAffinityTerm.GetMatchExpressions()) == 0 && len(nodeAffinityTerm.GetMatchFields()) == 0 { - glog.Warningf("NodeAffinityTerm %d is empty, skipping", i) + log.Warningf("NodeAffinityTerm %d is empty, skipping", i) continue } if nodeAffinityTerm.GetNodeAffinityJson() != nil { @@ -657,10 +660,10 @@ func extendPodSpecPatch( Weight: *nodeAffinityTerm.Weight, Preference: nodeSelectorTerm, }) - glog.V(4).Infof("Added preferred node affinity: %+v", nodeSelectorTerm) + log.Trace("Added preferred node affinity: %+v", nodeSelectorTerm) } else { requiredTerms = append(requiredTerms, nodeSelectorTerm) - glog.V(4).Infof("Added required node affinity: %+v", nodeSelectorTerm) + log.Trace("Added required node affinity: %+v", nodeSelectorTerm) } } @@ -729,18 +732,18 @@ func extendPodSpecPatch( isCompilerHardened := existingSecurityContext.AllowPrivilegeEscalation != nil && !*existingSecurityContext.AllowPrivilegeEscalation if userSecurityContext.RunAsUser != nil { if existingSecurityContext.RunAsUser != nil { - glog.Warningf("Ignoring user-specified runAsUser (%d): security context already set by admin (runAsUser=%d)", + log.Warningf("Ignoring user-specified runAsUser (%d): security context already set by admin (runAsUser=%d)", *userSecurityContext.RunAsUser, *existingSecurityContext.RunAsUser) } else { if isCompilerHardened && *userSecurityContext.RunAsUser == 0 { - glog.Warningf("Setting runAsUser=0 (root) on a container with hardened security context; consider using a non-root UID") + log.Warningf("Setting runAsUser=0 (root) on a container with hardened security context; consider using a non-root UID") } podSpec.Containers[0].SecurityContext.RunAsUser = userSecurityContext.RunAsUser } } if userSecurityContext.RunAsGroup != nil { if existingSecurityContext.RunAsGroup != nil { - glog.Warningf("Ignoring user-specified runAsGroup (%d): security context already set by admin (runAsGroup=%d)", + log.Warningf("Ignoring user-specified runAsGroup (%d): security context already set by admin (runAsGroup=%d)", *userSecurityContext.RunAsGroup, *existingSecurityContext.RunAsGroup) } else { podSpec.Containers[0].SecurityContext.RunAsGroup = userSecurityContext.RunAsGroup @@ -748,7 +751,7 @@ func extendPodSpecPatch( } if userSecurityContext.RunAsNonRoot != nil { if existingSecurityContext.RunAsNonRoot != nil { - glog.Warningf("Ignoring user-specified runAsNonRoot (%v): security context already set by admin (runAsNonRoot=%v)", + log.Warningf("Ignoring user-specified runAsNonRoot (%v): security context already set by admin (runAsNonRoot=%v)", *userSecurityContext.RunAsNonRoot, *existingSecurityContext.RunAsNonRoot) } else { podSpec.Containers[0].SecurityContext.RunAsNonRoot = userSecurityContext.RunAsNonRoot @@ -786,8 +789,13 @@ func createPVC( taskStartedTime := time.Now().Unix() + log := util.GetLoggerFrom(ctx) + if log == nil { + return "", nil, 0, fmt.Errorf("failed to get log from context") + } + inputs := execution.ExecutorInput.Inputs - glog.Infof("Input parameter values: %+v", inputs.ParameterValues) + log.Infof("Input parameter values: %+v", inputs.ParameterValues) // Required input: access_modes accessModeInput, ok := inputs.ParameterValues["access_modes"] @@ -867,7 +875,7 @@ func createPVC( if err != nil { return "", createdExecution, pb.Execution_FAILED, fmt.Errorf("error creating MLMD execution for createpvc: %w", err) } - glog.Infof("Created execution: %s", createdExecution) + log.Infof("Created execution: %s", createdExecution) execution.ID = createdExecution.GetID() if !execution.WillTrigger() { return "", createdExecution, pb.Execution_COMPLETE, nil @@ -917,7 +925,7 @@ func createPVC( if err != nil { return "", createdExecution, pb.Execution_FAILED, fmt.Errorf("failed to create pvc: %w", err) } - glog.Infof("Created PVC %s\n", createdPVC.ObjectMeta.Name) + log.Infof("Created PVC %s\n", createdPVC.ObjectMeta.Name) // Create a cache entry if !opts.CacheDisabled && opts.Task.GetCachingOptions().GetEnableCache() { @@ -951,9 +959,13 @@ func deletePVC( }() taskStartedTime := time.Now().Unix() + log := util.GetLoggerFrom(ctx) + if log == nil { + return nil, pb.Execution_FAILED, fmt.Errorf("no logs available for execution") + } inputs := execution.ExecutorInput.Inputs - glog.Infof("Input parameter values: %+v", inputs.ParameterValues) + log.Infof("Input parameter values: %+v", inputs.ParameterValues) // Required input: pvc_name pvcNameInput, ok := inputs.ParameterValues["pvc_name"] @@ -983,7 +995,7 @@ func deletePVC( if err != nil { return createdExecution, pb.Execution_FAILED, fmt.Errorf("error creating MLMD execution for createpvc: %w", err) } - glog.Infof("Created execution: %s", createdExecution) + log.Infof("Created execution: %s", createdExecution) execution.ID = createdExecution.GetID() if !execution.WillTrigger() { return createdExecution, pb.Execution_COMPLETE, nil @@ -1022,7 +1034,7 @@ func deletePVC( return createdExecution, pb.Execution_FAILED, fmt.Errorf("failed to delete pvc %s: %v", pvcName, err) } - glog.Infof("Deleted PVC %s\n", pvcName) + log.Infof("Deleted PVC %s\n", pvcName) // Create a cache entry if !opts.CacheDisabled && opts.Task.GetCachingOptions().GetEnableCache() && ecfg.CachedMLMDExecutionID != "" { @@ -1138,7 +1150,11 @@ func publishDriverExecution( if err = mlmd.PublishExecution(ctx, execution, outputParameters, outputArtifacts, status); err != nil { return fmt.Errorf("failed to publish: %w", err) } - glog.Infof("Published execution of Kubernetes platform task %s.", execution.TaskName()) + log := util.GetLoggerFrom(ctx) + if log == nil { + return fmt.Errorf("can not get log from the context ") + } + log.Infof("Published execution of Kubernetes platform task %s.", execution.TaskName()) return nil } diff --git a/backend/src/v2/driver/root_dag.go b/backend/src/v2/driver/root_dag.go index c017fcd7bbf..2e02b922d40 100644 --- a/backend/src/v2/driver/root_dag.go +++ b/backend/src/v2/driver/root_dag.go @@ -19,7 +19,6 @@ import ( "encoding/json" "fmt" - "github.com/golang/glog" "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" "github.com/kubeflow/pipelines/backend/src/common/util" "github.com/kubeflow/pipelines/backend/src/v2/config" @@ -64,54 +63,58 @@ func validateRootDAG(opts Options) (err error) { return nil } -func RootDAG(ctx context.Context, opts Options, mlmd *metadata.Client) (execution *Execution, err error) { +func RootDAG(ctx context.Context, opts Options, mlmd *metadata.Client) (execution *Execution, pipeline *metadata.Pipeline, err error) { defer func() { if err != nil { err = fmt.Errorf("driver.RootDAG(%s) failed: %w", opts.info(), err) } }() b, _ := json.Marshal(opts) - glog.V(4).Info("RootDAG opts: ", string(b)) + log := util.GetLoggerFrom(ctx) + if log == nil { + return nil, nil, fmt.Errorf("driver.RootDAG(%s) failed: invalid log configuration", opts.info()) + } + log.Trace("RootDAG opts: ", string(b)) err = validateRootDAG(opts) if err != nil { - return nil, err + return nil, nil, err } // TODO(v2): in pipeline spec, rename GCS output directory to pipeline root. pipelineRoot := opts.RuntimeConfig.GetGcsOutputDirectory() restConfig, err := util.GetKubernetesConfig() if err != nil { - return nil, fmt.Errorf("failed to initialize kubernetes client: %w", err) + return nil, nil, fmt.Errorf("failed to initialize kubernetes client: %w", err) } k8sClient, err := kubernetes.NewForConfig(restConfig) if err != nil { - return nil, fmt.Errorf("failed to initialize kubernetes client set: %w", err) + return nil, nil, fmt.Errorf("failed to initialize kubernetes client set: %w", err) } cfg, err := config.FromConfigMap(ctx, k8sClient, opts.Namespace) if err != nil { - return nil, err + return nil, nil, err } storeSessionInfo := objectstore.SessionInfo{} if pipelineRoot != "" { - glog.Infof("PipelineRoot=%q", pipelineRoot) + log.Infof("PipelineRoot=%q", pipelineRoot) } else { pipelineRoot = cfg.DefaultPipelineRoot() - glog.Infof("PipelineRoot=%q from default config", pipelineRoot) + log.Infof("PipelineRoot=%q from default config", pipelineRoot) } storeSessionInfo, err = cfg.GetStoreSessionInfo(pipelineRoot) if err != nil { - return nil, err + return nil, nil, err } storeSessionInfoJSON, err := json.Marshal(storeSessionInfo) if err != nil { - return nil, err + return nil, nil, err } storeSessionInfoStr := string(storeSessionInfoJSON) // TODO(Bobgy): fill in run resource. - pipeline, err := mlmd.GetPipeline(ctx, opts.PipelineName, opts.RunID, opts.Namespace, "run-resource", pipelineRoot, storeSessionInfoStr) + pipeline, err = mlmd.GetPipeline(ctx, opts.PipelineName, opts.RunID, opts.Namespace, "run-resource", pipelineRoot, storeSessionInfoStr) if err != nil { - return nil, err + return nil, nil, err } executorInput := &pipelinespec.ExecutorInput{ @@ -122,16 +125,16 @@ func RootDAG(ctx context.Context, opts Options, mlmd *metadata.Client) (executio // TODO(Bobgy): validate executorInput matches component spec types ecfg, err := metadata.GenerateExecutionConfig(executorInput) if err != nil { - return nil, err + return nil, pipeline, err } ecfg.ExecutionType = metadata.DagExecutionTypeName ecfg.Name = fmt.Sprintf("run/%s", opts.RunID) exec, err := mlmd.CreateExecution(ctx, pipeline, ecfg) if err != nil { - return nil, err + return nil, pipeline, err } - glog.Infof("Created execution: %s", exec) + log.Infof("Created execution: %s", exec) // No need to return ExecutorInput, because tasks in the DAG will resolve // needed info from MLMD. - return &Execution{ID: exec.GetID()}, nil + return &Execution{ID: exec.GetID()}, pipeline, nil } diff --git a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py index ede1011c49b..3016c0abfb6 100644 --- a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py +++ b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py @@ -175,6 +175,48 @@ def upsert_lifecycle_policy(self, bucket_name, artifact_retention_days): else: print(f"ERROR: Failed to configure lifecycle policy: {exception}") + def create_role_and_binding(self, rbac_v1, namespace, role_name, sa_name, resources, verbs, resource_names=None): + # Configures necessary roles for the driver executor plugin + role = client.V1Role( + metadata=client.V1ObjectMeta( + name=role_name, + namespace=namespace + ), + rules=[ + client.V1PolicyRule( + api_groups=[""], + resources=resources, + resource_names=resource_names, + verbs=verbs + ) + ] + ) + rbac_v1.create_namespaced_role(namespace=namespace, body=role) + print(f"Role {role_name} created in {namespace}") + + role_binding_name = f"{role_name}-binding" + role_binding = client.V1RoleBinding( + metadata=client.V1ObjectMeta( + name=role_binding_name, + namespace=namespace + ), + subjects=[ + { + "kind": "ServiceAccount", + "name": sa_name, + "namespace": namespace, + "apiGroup": "" + } + ], + role_ref=client.V1RoleRef( + kind="Role", + name=role_name, + api_group="rbac.authorization.k8s.io" + ) + ) + rbac_v1.create_namespaced_role_binding(namespace=namespace, body=role_binding) + print(f"RoleBinding {role_binding_name} created in {namespace}") + def upsert_executor_plugin_sa(self, namespace): print('create executor plugin SAs for: ', namespace) try: @@ -195,52 +237,10 @@ def upsert_executor_plugin_sa(self, namespace): ) ) print(f"ServiceAccount {agent_sa_name} created in {namespace}") - role_name = "configmap-reader" - role = client.V1Role( - metadata=client.V1ObjectMeta( - name=role_name, - namespace=namespace - ), - rules=[ - client.V1PolicyRule( - api_groups=[""], - resources=["configmaps"], - verbs=["get", "list", "watch"] - ) - ] - ) - rbac_v1.create_namespaced_role(namespace=namespace, body=role) - print(f"Role {role_name} created in {namespace}") - role_binding_name = "configmap-reader-binding" - namespace = 'kubeflow-user-example-com' agent_sa_name = "ml-pipeline-driver-agent-executor-plugin" - role_name = "configmap-reader" - role_binding = client.V1RoleBinding( - metadata=client.V1ObjectMeta( - name="configmap-reader-binding", - namespace=namespace - ), - subjects=[ - { - "kind": "ServiceAccount", - "name": agent_sa_name, - "namespace": namespace, - "apiGroup": "" - } - ], - role_ref=client.V1RoleRef( - kind="Role", - name=role_name, - api_group="rbac.authorization.k8s.io" - ) - ) - - rbac_v1.create_namespaced_role_binding( - namespace=namespace, - body=role_binding - ) - print(f"RoleBinding {role_binding_name} created in {namespace}") + self.create_role_and_binding(rbac_v1=rbac_v1, namespace=namespace, role_name='configmap-reader', resources=["configmaps"], sa_name=agent_sa_name, verbs=["get", "list", "watch"]) + self.create_role_and_binding(rbac_v1=rbac_v1, namespace=namespace, role_name='artifact-secret-reader', resources=["secrets"], sa_name=agent_sa_name, verbs=["get", "list", "watch"], resource_names=["mlpipeline-minio-artifact"]) except ApiException as e: if e.status == 409: print(f"ServiceAccount {agent_sa_name} already exists in {namespace}") diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml index aca2f9fb22e..17756563d51 100644 --- a/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml +++ b/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml @@ -9,6 +9,18 @@ data: sidecar.container: | name: driver-plugin image: ghcr.io/kubeflow/kfp-driver:dummy + imagePullPolicy: IfNotPresent + env: + - name: LOG_ACCESS_KEY + valueFrom: + secretKeyRef: + name: mlpipeline-minio-artifact + key: accesskey + - name: LOG_SECRET_KEY + valueFrom: + secretKeyRef: + name: mlpipeline-minio-artifact + key: secretkey ports: - containerPort: 8080 resources: @@ -27,3 +39,7 @@ data: - ALL seccompProfile: type: RuntimeDefault + volumeMounts: + - name: var-run-argo + mountPath: /kfp/log + readOnly: false diff --git a/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml b/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml index e76cba5b2e0..23ad90b1bf2 100644 --- a/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml +++ b/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml @@ -129,4 +129,30 @@ subjects: roleRef: kind: Role name: configmap-reader + apiGroup: rbac.authorization.k8s.io + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: artifact-secret-reader + namespace: kubeflow +rules: + - apiGroups: [""] + resources: ["secrets"] + resourceNames: ["mlpipeline-minio-artifact"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: artifact-secret-reader-binding + namespace: kubeflow +subjects: + - kind: ServiceAccount + name: ml-pipeline-driver-agent-executor-plugin + namespace: kubeflow +roleRef: + kind: Role + name: artifact-secret-reader apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-driver-plugin-cm.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-driver-plugin-cm.yaml index 06cc855769d..02072786ecf 100644 --- a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-driver-plugin-cm.yaml +++ b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-driver-plugin-cm.yaml @@ -9,6 +9,17 @@ data: sidecar.container: | name: driver-plugin image: ghcr.io/kubeflow/kfp-driver:dummy + env: + - name: LOG_ACCESS_KEY + valueFrom: + secretKeyRef: + name: mlpipeline-minio-artifact + key: accesskey + - name: LOG_SECRET_KEY + valueFrom: + secretKeyRef: + name: mlpipeline-minio-artifact + key: secretkey ports: - containerPort: 8080 resources: @@ -31,3 +42,6 @@ data: - name: argo-workflows-agent-ca-certificates mountPath: /kfp/certs readOnly: true + - name: var-run-argo + mountPath: /kfp/log + readOnly: false diff --git a/manifests/kustomize/env/dev/driver-plugin-cm-path.yaml b/manifests/kustomize/env/dev/driver-plugin-cm-path.yaml index 049d2b09a14..f6918cec176 100644 --- a/manifests/kustomize/env/dev/driver-plugin-cm-path.yaml +++ b/manifests/kustomize/env/dev/driver-plugin-cm-path.yaml @@ -6,6 +6,17 @@ data: sidecar.container: | name: driver-plugin image: ghcr.io/kubeflow/kfp-driver:master + env: + - name: LOG_ACCESS_KEY + valueFrom: + secretKeyRef: + name: mlpipeline-minio-artifact + key: accesskey + - name: LOG_SECRET_KEY + valueFrom: + secretKeyRef: + name: mlpipeline-minio-artifact + key: secretkey ports: - containerPort: 8080 resources: @@ -23,4 +34,8 @@ data: drop: - ALL seccompProfile: - type: RuntimeDefault \ No newline at end of file + type: RuntimeDefault + volumeMounts: + - name: var-run-argo + mountPath: /kfp/log + readOnly: false \ No newline at end of file From b422c5583c95ae5d076360022fc1fecce3a276ad Mon Sep 17 00:00:00 2001 From: arpechenin Date: Mon, 9 Mar 2026 21:50:22 +0300 Subject: [PATCH 33/51] Store driver logs in artifact storage and display container driver logs in UI Signed-off-by: arpechenin --- backend/src/driver/rpc_handler.go | 4 +- backend/src/v2/driver/container.go | 2 + backend/src/v2/driver/driver.go | 21 ++--- backend/src/v2/metadata/client.go | 6 ++ .../components/tabs/RuntimeNodeDetailsV2.tsx | 79 ++++++++++++++++++- 5 files changed, 94 insertions(+), 18 deletions(-) diff --git a/backend/src/driver/rpc_handler.go b/backend/src/driver/rpc_handler.go index 4b39ec48a0b..172bd2314c9 100644 --- a/backend/src/driver/rpc_handler.go +++ b/backend/src/driver/rpc_handler.go @@ -63,7 +63,7 @@ func ExecutePlugin(w http.ResponseWriter, r *http.Request) { http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) return } - + glog.Infof("Received request to execute plugin: %v", r) args, err := parseDriverRequestArgs(r) if err != nil { glog.Errorf("Failed to parse driver request args: %v", err) @@ -75,7 +75,6 @@ func ExecutePlugin(w http.ResponseWriter, r *http.Request) { http.Error(w, "Driver plugin requires at least one argument", http.StatusBadRequest) return } - glog.Infof("driver plugin arguments: %v", args) execution, err := drive(*args) outputs := extractOutputParameters(execution, args.Type) if err != nil { @@ -197,6 +196,7 @@ func drive(args api.DriverPluginArgs) (execution *driver.Execution, err error) { log := util.GetLoggerFrom(ctx) + log.Infof("driver plugin arguments: %v", args) // Support reading component spec from a file if value starts with @ // This bypasses exec() argument size limits for large workflows if strings.HasPrefix(args.Component, "@") { diff --git a/backend/src/v2/driver/container.go b/backend/src/v2/driver/container.go index 1383a803b5e..d7449511608 100644 --- a/backend/src/v2/driver/container.go +++ b/backend/src/v2/driver/container.go @@ -125,11 +125,13 @@ func Container(ctx context.Context, pipeline *metadata.Pipeline, opts Options, m return execution, err } ecfg.TaskName = opts.TaskName + ecfg.Namespace = opts.Namespace ecfg.DisplayName = opts.Task.GetTaskInfo().GetName() ecfg.ExecutionType = metadata.ContainerExecutionTypeName ecfg.ParentDagID = dag.Execution.GetID() ecfg.IterationIndex = iterationIndex ecfg.NotTriggered = !execution.WillTrigger() + ecfg.DriverLogUri = metadata.GenerateOutputURI(pipeline.GetPipelineRoot(), []string{opts.TaskName, outputPathPrefix, "driver-logs"}, false) if isKubernetesPlatformOp { return execution, kubernetesPlatformOps(ctx, mlmd, cacheClient, execution, ecfg, &opts) diff --git a/backend/src/v2/driver/driver.go b/backend/src/v2/driver/driver.go index ab12434b9df..c0d34c25474 100644 --- a/backend/src/v2/driver/driver.go +++ b/backend/src/v2/driver/driver.go @@ -712,21 +712,13 @@ func provisionOutputs( if artifacts == nil { artifacts = make(map[string]*pipelinespec.ComponentOutputsSpec_ArtifactSpec) } - artifacts["executor-logs"] = &pipelinespec.ComponentOutputsSpec_ArtifactSpec{ - ArtifactType: &pipelinespec.ArtifactTypeSchema{ - Kind: &pipelinespec.ArtifactTypeSchema_SchemaTitle{ - SchemaTitle: "system.Artifact", - }, - }, - } - artifacts["driver-logs"] = &pipelinespec.ComponentOutputsSpec_ArtifactSpec{ - ArtifactType: &pipelinespec.ArtifactTypeSchema{ - Kind: &pipelinespec.ArtifactTypeSchema_SchemaTitle{ - SchemaTitle: "system.Artifact", - }, + } + artifacts["executor-logs"] = &pipelinespec.ComponentOutputsSpec_ArtifactSpec{ + ArtifactType: &pipelinespec.ArtifactTypeSchema{ + Kind: &pipelinespec.ArtifactTypeSchema_SchemaTitle{ + SchemaTitle: "system.Artifact", }, - } - + }, } // Compute a task-root remote URI that will serve as the base for all @@ -767,7 +759,6 @@ func provisionOutputs( // Fallback to legacy path if the pipeline root scheme is not recognized. outputs.OutputFile = component.OutputMetadataFilepath } - return outputs } diff --git a/backend/src/v2/metadata/client.go b/backend/src/v2/metadata/client.go index e0b9f76a77d..86eee3ceacc 100644 --- a/backend/src/v2/metadata/client.go +++ b/backend/src/v2/metadata/client.go @@ -181,6 +181,7 @@ type ExecutionConfig struct { OutputArtifacts map[string]*pipelinespec.DagOutputsSpec_DagOutputArtifactSpec InputArtifactIDs map[string][]int64 IterationIndex *int // Index of the iteration. + DriverLogUri string // ContainerExecution custom properties Image, CachedMLMDExecutionID, FingerPrint string @@ -577,6 +578,7 @@ const ( keyOutputs = "outputs" keyParameterProducerTask = "parameter_producer_task" keyOutputArtifacts = "output_artifacts" + keyDriverLogUri = "driver_logs_uri" keyArtifactProducerTask = "artifact_producer_task" keyParentDagID = "parent_dag_id" // Parent DAG Execution ID. keyIterationIndex = "iteration_index" @@ -634,6 +636,10 @@ func (c *Client) CreateExecution(ctx context.Context, pipeline *Pipeline, config e.CustomProperties[keyCacheFingerPrint] = StringValue(config.FingerPrint) } } + if config.DriverLogUri != "" { + e.CustomProperties[keyDriverLogUri] = StringValue(config.DriverLogUri) + e.CustomProperties[keyStoreSessionInfo] = StringValue(pipeline.GetStoreSessionInfo()) + } if config.InputParameters != nil { e.CustomProperties[keyInputs] = &pb.Value{Value: &pb.Value_StructValue{ StructValue: &structpb.Struct{ diff --git a/frontend/src/components/tabs/RuntimeNodeDetailsV2.tsx b/frontend/src/components/tabs/RuntimeNodeDetailsV2.tsx index f92adb30395..c77cdcca975 100644 --- a/frontend/src/components/tabs/RuntimeNodeDetailsV2.tsx +++ b/frontend/src/components/tabs/RuntimeNodeDetailsV2.tsx @@ -63,6 +63,9 @@ import { getComponentSpec } from 'src/lib/v2/NodeUtils'; export const LOGS_DETAILS = 'logs_details'; export const LOGS_BANNER_MESSAGE = 'logs_banner_message'; export const LOGS_BANNER_ADDITIONAL_INFO = 'logs_banner_additional_info'; +export const SYS_LOGS_DETAILS = 'sys_log_details'; +export const SYS_LOGS_BANNER_MESSAGE = 'sys_logs_banner_message'; +export const SYS_LOGS_BANNER_ADDITIONAL_INFO = 'sys_logs_banner_additional_info'; export const K8S_PLATFORM_KEY = 'kubernetes'; const NODE_INFO_UNKNOWN = ( @@ -167,16 +170,30 @@ function TaskNodeDetail({ { enabled: !!execution }, ); + const { data: driverLogsInfo } = useQuery, Error>( + ['driver-logs', { executionId: execution?.getId(), namespace }], + async () => { + if (!execution) { + throw new Error('No execution is found.'); + } + return await getDriverLogsInfo(execution, namespace); + }, + { enabled: !!execution }, + ); + const logsDetails = logsInfo?.get(LOGS_DETAILS); const logsBannerMessage = logsInfo?.get(LOGS_BANNER_MESSAGE); const logsBannerAdditionalInfo = logsInfo?.get(LOGS_BANNER_ADDITIONAL_INFO); + const sysLogDetails = driverLogsInfo?.get(SYS_LOGS_DETAILS); + const sysLogsBannerMessage = driverLogsInfo?.get(SYS_LOGS_BANNER_MESSAGE); + const sysLogsBannerAdditionalInfo = driverLogsInfo?.get(SYS_LOGS_BANNER_ADDITIONAL_INFO); const [selectedTab, setSelectedTab] = useState(0); return (
setSelectedTab(tab)} /> @@ -215,6 +232,21 @@ function TaskNodeDetail({ )}
)} + {/* System Logs tab */} + {selectedTab === 3 && ( +
+ {sysLogsBannerMessage && ( + + + + )} + {!sysLogsBannerMessage && ( +
+ +
+ )} +
+ )} ); @@ -302,6 +334,51 @@ function getNodeVolumeMounts( return volumeMounts; } +async function getDriverLogsInfo( + execution: Execution, + namespace?: string, +): Promise> { + const logsInfo = new Map(); + let podNameSpace = ''; + let logsDetails = ''; + let logsBannerMessage = ''; + let logsBannerAdditionalInfo = ''; + const customPropertiesMap = execution.getCustomPropertiesMap(); + + if (execution) { + podNameSpace = customPropertiesMap.get('namespace')?.getStringValue() || ''; + } + + try { + const driverLogUri = customPropertiesMap.get('driver_logs_uri')?.getStringValue(); + const storeSessionInfo = customPropertiesMap.get('store_session_info')?.getStringValue(); + + if (driverLogUri && storeSessionInfo) { + const storagePath = WorkflowParser.parseStoragePath(driverLogUri); + const providerInfo = storeSessionInfo; + const artifactNamespace = namespace || podNameSpace; + + logsDetails = await Apis.readFile({ + path: storagePath, + providerInfo: providerInfo, + namespace: artifactNamespace, + }); + logsInfo.set(SYS_LOGS_DETAILS, logsDetails); + return logsInfo; + } + } catch (artifactErr) { + let errMsg = await errorToMessage(artifactErr); + logsBannerMessage = 'Failed to retrieve pod logs.'; + logsInfo.set(SYS_LOGS_BANNER_MESSAGE, logsBannerMessage); + logsBannerAdditionalInfo = 'Error response: ' + errMsg; + logsInfo.set(SYS_LOGS_BANNER_ADDITIONAL_INFO, logsBannerAdditionalInfo); + + console.error('Failed to retrieve driver-logs artifact:', artifactErr); + } + + return logsInfo; +} + async function getLogsInfo( execution: Execution, runId?: string, From aa1abdb880d211ae319ea796e1394426cc0c7fd9 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Mon, 9 Mar 2026 21:53:30 +0300 Subject: [PATCH 34/51] apply formatting to frontend Signed-off-by: arpechenin --- .../components/tabs/RuntimeNodeDetailsV2.tsx | 119 +++++++++--------- 1 file changed, 61 insertions(+), 58 deletions(-) diff --git a/frontend/src/components/tabs/RuntimeNodeDetailsV2.tsx b/frontend/src/components/tabs/RuntimeNodeDetailsV2.tsx index c77cdcca975..3b721459193 100644 --- a/frontend/src/components/tabs/RuntimeNodeDetailsV2.tsx +++ b/frontend/src/components/tabs/RuntimeNodeDetailsV2.tsx @@ -170,16 +170,16 @@ function TaskNodeDetail({ { enabled: !!execution }, ); - const { data: driverLogsInfo } = useQuery, Error>( - ['driver-logs', { executionId: execution?.getId(), namespace }], - async () => { - if (!execution) { - throw new Error('No execution is found.'); - } - return await getDriverLogsInfo(execution, namespace); - }, - { enabled: !!execution }, - ); + const { data: driverLogsInfo } = useQuery, Error>( + ['driver-logs', { executionId: execution?.getId(), namespace }], + async () => { + if (!execution) { + throw new Error('No execution is found.'); + } + return await getDriverLogsInfo(execution, namespace); + }, + { enabled: !!execution }, + ); const logsDetails = logsInfo?.get(LOGS_DETAILS); const logsBannerMessage = logsInfo?.get(LOGS_BANNER_MESSAGE); @@ -234,18 +234,21 @@ function TaskNodeDetail({ )} {/* System Logs tab */} {selectedTab === 3 && ( -
- {sysLogsBannerMessage && ( - - - - )} - {!sysLogsBannerMessage && ( -
- -
- )} +
+ {sysLogsBannerMessage && ( + + + + )} + {!sysLogsBannerMessage && ( +
+
+ )} +
)}
@@ -335,48 +338,48 @@ function getNodeVolumeMounts( } async function getDriverLogsInfo( - execution: Execution, - namespace?: string, + execution: Execution, + namespace?: string, ): Promise> { - const logsInfo = new Map(); - let podNameSpace = ''; - let logsDetails = ''; - let logsBannerMessage = ''; - let logsBannerAdditionalInfo = ''; - const customPropertiesMap = execution.getCustomPropertiesMap(); - - if (execution) { - podNameSpace = customPropertiesMap.get('namespace')?.getStringValue() || ''; - } + const logsInfo = new Map(); + let podNameSpace = ''; + let logsDetails = ''; + let logsBannerMessage = ''; + let logsBannerAdditionalInfo = ''; + const customPropertiesMap = execution.getCustomPropertiesMap(); - try { - const driverLogUri = customPropertiesMap.get('driver_logs_uri')?.getStringValue(); - const storeSessionInfo = customPropertiesMap.get('store_session_info')?.getStringValue(); - - if (driverLogUri && storeSessionInfo) { - const storagePath = WorkflowParser.parseStoragePath(driverLogUri); - const providerInfo = storeSessionInfo; - const artifactNamespace = namespace || podNameSpace; - - logsDetails = await Apis.readFile({ - path: storagePath, - providerInfo: providerInfo, - namespace: artifactNamespace, - }); - logsInfo.set(SYS_LOGS_DETAILS, logsDetails); - return logsInfo; - } - } catch (artifactErr) { - let errMsg = await errorToMessage(artifactErr); - logsBannerMessage = 'Failed to retrieve pod logs.'; - logsInfo.set(SYS_LOGS_BANNER_MESSAGE, logsBannerMessage); - logsBannerAdditionalInfo = 'Error response: ' + errMsg; - logsInfo.set(SYS_LOGS_BANNER_ADDITIONAL_INFO, logsBannerAdditionalInfo); + if (execution) { + podNameSpace = customPropertiesMap.get('namespace')?.getStringValue() || ''; + } - console.error('Failed to retrieve driver-logs artifact:', artifactErr); + try { + const driverLogUri = customPropertiesMap.get('driver_logs_uri')?.getStringValue(); + const storeSessionInfo = customPropertiesMap.get('store_session_info')?.getStringValue(); + + if (driverLogUri && storeSessionInfo) { + const storagePath = WorkflowParser.parseStoragePath(driverLogUri); + const providerInfo = storeSessionInfo; + const artifactNamespace = namespace || podNameSpace; + + logsDetails = await Apis.readFile({ + path: storagePath, + providerInfo: providerInfo, + namespace: artifactNamespace, + }); + logsInfo.set(SYS_LOGS_DETAILS, logsDetails); + return logsInfo; } + } catch (artifactErr) { + let errMsg = await errorToMessage(artifactErr); + logsBannerMessage = 'Failed to retrieve pod logs.'; + logsInfo.set(SYS_LOGS_BANNER_MESSAGE, logsBannerMessage); + logsBannerAdditionalInfo = 'Error response: ' + errMsg; + logsInfo.set(SYS_LOGS_BANNER_ADDITIONAL_INFO, logsBannerAdditionalInfo); + + console.error('Failed to retrieve driver-logs artifact:', artifactErr); + } - return logsInfo; + return logsInfo; } async function getLogsInfo( From c70b8552fec123558b940c792f7c82a57c98d1a7 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Mon, 9 Mar 2026 22:10:30 +0300 Subject: [PATCH 35/51] fix linter errors Signed-off-by: arpechenin --- backend/src/driver/rpc_handler.go | 14 +++++--------- backend/src/v2/driver/container.go | 2 +- backend/src/v2/driver/dag.go | 2 +- backend/src/v2/driver/k8s.go | 6 +++--- backend/src/v2/metadata/client.go | 8 ++++---- 5 files changed, 14 insertions(+), 18 deletions(-) diff --git a/backend/src/driver/rpc_handler.go b/backend/src/driver/rpc_handler.go index 172bd2314c9..c44ebf9ac62 100644 --- a/backend/src/driver/rpc_handler.go +++ b/backend/src/driver/rpc_handler.go @@ -48,7 +48,7 @@ type driverLogArtifactContext struct { Namespace string PipelineRoot string StoreSessionInfo string - LogId string + LogID string } func ExecutePlugin(w http.ResponseWriter, r *http.Request) { @@ -154,9 +154,9 @@ func drive(args api.DriverPluginArgs) (execution *driver.Execution, err error) { outputPathPrefix string ) var pipeline *metadata.Pipeline - logId := fmt.Sprintf("%v-%v-%v", args.IterationIndex, args.Type, args.TaskName) + logID := fmt.Sprintf("%v-%v-%v", args.IterationIndex, args.Type, args.TaskName) logDir := "/kfp/log" - logFile := fmt.Sprintf("%s/%s.log", logDir, logId) + logFile := fmt.Sprintf("%s/%s.log", logDir, logID) ctx, f, err := util.WithLogger(context.Background(), logFile) if err != nil { return nil, fmt.Errorf("failed to create driver logger: %v", err) @@ -173,7 +173,7 @@ func drive(args api.DriverPluginArgs) (execution *driver.Execution, err error) { Execution: execution, Task: args.TaskName, LocalPath: logFile, - LogId: logId, + LogID: logID, Namespace: namespace, PipelineRoot: pipelineRoot, StoreSessionInfo: storeSessionInfo, @@ -384,7 +384,7 @@ func uploadDriverLogArtifact(ctx context.Context, logContext *driverLogArtifactC if err != nil { return fmt.Errorf("failed to open bucket: %v", err) } - key := fmt.Sprintf("driver/%s-logs", logContext.LogId) + key := fmt.Sprintf("driver/%s-logs", logContext.LogID) if logContext.Execution != nil && logContext.OutputPathPrefix != "" { key = fmt.Sprintf("%s/%s/driver-logs", logContext.Task, logContext.OutputPathPrefix) } @@ -460,7 +460,3 @@ func WriteJSONResponse(w http.ResponseWriter, payload api.DriverResponse) { http.Error(w, "failed to encode response", http.StatusInternalServerError) } } - -func putLog() { - -} diff --git a/backend/src/v2/driver/container.go b/backend/src/v2/driver/container.go index d7449511608..bef4fee3340 100644 --- a/backend/src/v2/driver/container.go +++ b/backend/src/v2/driver/container.go @@ -131,7 +131,7 @@ func Container(ctx context.Context, pipeline *metadata.Pipeline, opts Options, m ecfg.ParentDagID = dag.Execution.GetID() ecfg.IterationIndex = iterationIndex ecfg.NotTriggered = !execution.WillTrigger() - ecfg.DriverLogUri = metadata.GenerateOutputURI(pipeline.GetPipelineRoot(), []string{opts.TaskName, outputPathPrefix, "driver-logs"}, false) + ecfg.DriverLogURI = metadata.GenerateOutputURI(pipeline.GetPipelineRoot(), []string{opts.TaskName, outputPathPrefix, "driver-logs"}, false) if isKubernetesPlatformOp { return execution, kubernetesPlatformOps(ctx, mlmd, cacheClient, execution, ecfg, &opts) diff --git a/backend/src/v2/driver/dag.go b/backend/src/v2/driver/dag.go index d494610c872..39464abe2e5 100644 --- a/backend/src/v2/driver/dag.go +++ b/backend/src/v2/driver/dag.go @@ -165,7 +165,7 @@ func DAG(ctx context.Context, pipeline *metadata.Pipeline, opts Options, mlmd *m log.Trace("pipeline: ", pipeline) b, _ = json.Marshal(*ecfg) log.Trace("ecfg: ", string(b)) - log.Trace("dag: %v", dag) + log.Tracef("dag: %v", dag) // TODO(Bobgy): change execution state to pending, because this is driver, execution hasn't started. createdExecution, err := mlmd.CreateExecution(ctx, pipeline, ecfg) diff --git a/backend/src/v2/driver/k8s.go b/backend/src/v2/driver/k8s.go index ae27d990f30..af3088a3d52 100644 --- a/backend/src/v2/driver/k8s.go +++ b/backend/src/v2/driver/k8s.go @@ -660,10 +660,10 @@ func extendPodSpecPatch( Weight: *nodeAffinityTerm.Weight, Preference: nodeSelectorTerm, }) - log.Trace("Added preferred node affinity: %+v", nodeSelectorTerm) + log.Tracef("Added preferred node affinity: %+v", nodeSelectorTerm) } else { requiredTerms = append(requiredTerms, nodeSelectorTerm) - log.Trace("Added required node affinity: %+v", nodeSelectorTerm) + log.Tracef("Added required node affinity: %+v", nodeSelectorTerm) } } @@ -925,7 +925,7 @@ func createPVC( if err != nil { return "", createdExecution, pb.Execution_FAILED, fmt.Errorf("failed to create pvc: %w", err) } - log.Infof("Created PVC %s\n", createdPVC.ObjectMeta.Name) + log.Infof("Created PVC %s\n", createdPVC.Name) // Create a cache entry if !opts.CacheDisabled && opts.Task.GetCachingOptions().GetEnableCache() { diff --git a/backend/src/v2/metadata/client.go b/backend/src/v2/metadata/client.go index 86eee3ceacc..e10e7fb056f 100644 --- a/backend/src/v2/metadata/client.go +++ b/backend/src/v2/metadata/client.go @@ -181,7 +181,7 @@ type ExecutionConfig struct { OutputArtifacts map[string]*pipelinespec.DagOutputsSpec_DagOutputArtifactSpec InputArtifactIDs map[string][]int64 IterationIndex *int // Index of the iteration. - DriverLogUri string + DriverLogURI string // ContainerExecution custom properties Image, CachedMLMDExecutionID, FingerPrint string @@ -578,7 +578,7 @@ const ( keyOutputs = "outputs" keyParameterProducerTask = "parameter_producer_task" keyOutputArtifacts = "output_artifacts" - keyDriverLogUri = "driver_logs_uri" + keyDriverLogURI = "driver_logs_uri" keyArtifactProducerTask = "artifact_producer_task" keyParentDagID = "parent_dag_id" // Parent DAG Execution ID. keyIterationIndex = "iteration_index" @@ -636,8 +636,8 @@ func (c *Client) CreateExecution(ctx context.Context, pipeline *Pipeline, config e.CustomProperties[keyCacheFingerPrint] = StringValue(config.FingerPrint) } } - if config.DriverLogUri != "" { - e.CustomProperties[keyDriverLogUri] = StringValue(config.DriverLogUri) + if config.DriverLogURI != "" { + e.CustomProperties[keyDriverLogURI] = StringValue(config.DriverLogURI) e.CustomProperties[keyStoreSessionInfo] = StringValue(pipeline.GetStoreSessionInfo()) } if config.InputParameters != nil { From 1ae6c97fc5a764e7f79b06abab8241b69eaa6e48 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Mon, 9 Mar 2026 22:40:13 +0300 Subject: [PATCH 36/51] fix tests Signed-off-by: arpechenin --- backend/src/common/util/context_logger.go | 5 +++ backend/src/v2/driver/driver.go | 12 +++--- backend/src/v2/driver/driver_test.go | 6 ++- backend/src/v2/driver/k8s_test.go | 48 ++++++++++++----------- 4 files changed, 40 insertions(+), 31 deletions(-) diff --git a/backend/src/common/util/context_logger.go b/backend/src/common/util/context_logger.go index 2c41a25ec76..0240e819dde 100644 --- a/backend/src/common/util/context_logger.go +++ b/backend/src/common/util/context_logger.go @@ -27,6 +27,11 @@ func newFileLogger(logFile string) (*logrus.Logger, io.Closer, error) { return logger, f, nil } +// WithExistingLogger For testing only +func WithExistingLogger(ctx context.Context, logger *logrus.Logger) context.Context { + return context.WithValue(ctx, contextLoggerKey, logger) +} + func WithLogger(ctx context.Context, logFile string) (context.Context, io.Closer, error) { if ctx == nil { return nil, nil, fmt.Errorf( diff --git a/backend/src/v2/driver/driver.go b/backend/src/v2/driver/driver.go index c0d34c25474..033daee5a99 100644 --- a/backend/src/v2/driver/driver.go +++ b/backend/src/v2/driver/driver.go @@ -712,13 +712,13 @@ func provisionOutputs( if artifacts == nil { artifacts = make(map[string]*pipelinespec.ComponentOutputsSpec_ArtifactSpec) } - } - artifacts["executor-logs"] = &pipelinespec.ComponentOutputsSpec_ArtifactSpec{ - ArtifactType: &pipelinespec.ArtifactTypeSchema{ - Kind: &pipelinespec.ArtifactTypeSchema_SchemaTitle{ - SchemaTitle: "system.Artifact", + artifacts["executor-logs"] = &pipelinespec.ComponentOutputsSpec_ArtifactSpec{ + ArtifactType: &pipelinespec.ArtifactTypeSchema{ + Kind: &pipelinespec.ArtifactTypeSchema_SchemaTitle{ + SchemaTitle: "system.Artifact", + }, }, - }, + } } // Compute a task-root remote URI that will serve as the base for all diff --git a/backend/src/v2/driver/driver_test.go b/backend/src/v2/driver/driver_test.go index e0a1f9d0858..a7b39fe529f 100644 --- a/backend/src/v2/driver/driver_test.go +++ b/backend/src/v2/driver/driver_test.go @@ -19,6 +19,8 @@ import ( "fmt" "testing" + "github.com/kubeflow/pipelines/backend/src/common/util" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" "github.com/kubeflow/pipelines/backend/src/apiserver/config/proxy" @@ -1469,7 +1471,7 @@ func Test_initPodSpecPatch_TaskConfig_Affinity_NodeSelector_Tolerations_Passthro assert.Nil(t, err) err = extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), podSpec, opts, nil, @@ -1572,7 +1574,7 @@ func Test_initPodSpecPatch_TaskConfig_Affinity_NodeSelector_Tolerations_ApplyAnd assert.Nil(t, err) err = extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), podSpec, opts, nil, diff --git a/backend/src/v2/driver/k8s_test.go b/backend/src/v2/driver/k8s_test.go index 884f448a656..2225063600f 100644 --- a/backend/src/v2/driver/k8s_test.go +++ b/backend/src/v2/driver/k8s_test.go @@ -5,8 +5,10 @@ import ( "testing" "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" + "github.com/kubeflow/pipelines/backend/src/common/util" "github.com/kubeflow/pipelines/backend/src/v2/metadata" "github.com/kubeflow/pipelines/kubernetes_platform/go/kubernetesplatform" + "github.com/sirupsen/logrus" "github.com/spf13/viper" "github.com/stretchr/testify/assert" "google.golang.org/protobuf/types/known/structpb" @@ -102,7 +104,7 @@ func Test_makeVolumeMountPatch(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { volumeMounts, volumes, err := makeVolumeMountPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), Options{}, tt.args.pvcMount, tt.args.dag, @@ -238,7 +240,7 @@ func Test_makePodSpecPatch_nodeSelector(t *testing.T) { taskConfig := &TaskConfig{} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -717,7 +719,7 @@ func Test_extendPodSpecPatch_Secret(t *testing.T) { taskConfig := &TaskConfig{} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), tt.podSpec, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -1214,7 +1216,7 @@ func Test_extendPodSpecPatch_ConfigMap(t *testing.T) { taskConfig := &TaskConfig{} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), tt.podSpec, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -1388,7 +1390,7 @@ func Test_extendPodSpecPatch_EmptyVolumeMount(t *testing.T) { taskConfig := &TaskConfig{} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), tt.podSpec, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -1514,7 +1516,7 @@ func Test_extendPodSpecPatch_ImagePullSecrets(t *testing.T) { }, }} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -1953,7 +1955,7 @@ func Test_extendPodSpecPatch_Tolerations(t *testing.T) { taskConfig := &TaskConfig{} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -2060,7 +2062,7 @@ func Test_extendPodSpecPatch_FieldPathAsEnv(t *testing.T) { taskConfig := &TaskConfig{} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -2133,7 +2135,7 @@ func Test_extendPodSpecPatch_ActiveDeadlineSeconds(t *testing.T) { }, }} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -2246,7 +2248,7 @@ func Test_extendPodSpecPatch_SecurityContext(t *testing.T) { }, }} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -2274,7 +2276,7 @@ func Test_extendPodSpecPatch_SecurityContext_CombinedWithOtherFeatures(t *testin }, }} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{KubernetesExecutorConfig: &kubernetesplatform.KubernetesExecutorConfig{ SecurityContext: &kubernetesplatform.SecurityContext{ @@ -2321,7 +2323,7 @@ func Test_extendPodSpecPatch_SecurityContext_AdminSetPreserved(t *testing.T) { {Name: "main"}, }} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{ DefaultRunAsUser: &adminUID, @@ -2355,7 +2357,7 @@ func Test_extendPodSpecPatch_SecurityContext_AdminDefaultsNoUserOverride(t *test {Name: "main"}, }} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{ DefaultRunAsUser: &adminUID, @@ -2383,7 +2385,7 @@ func Test_extendPodSpecPatch_SecurityContext_RootOnHardenedContainer(t *testing. }, }} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{KubernetesExecutorConfig: &kubernetesplatform.KubernetesExecutorConfig{ SecurityContext: &kubernetesplatform.SecurityContext{ @@ -2424,7 +2426,7 @@ func Test_extendPodSpecPatch_SecurityContext_AdminRunAsNonRoot(t *testing.T) { {Name: "main"}, }} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{ DefaultRunAsNonRoot: &adminRunAsNonRoot, @@ -2453,7 +2455,7 @@ func Test_extendPodSpecPatch_SecurityContext_AdminRunAsNonRootNoUserOverride(t * {Name: "main"}, }} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{ DefaultRunAsNonRoot: &adminRunAsNonRoot, @@ -2475,7 +2477,7 @@ func Test_extendPodSpecPatch_SecurityContext_UserRunAsNonRootNoAdmin(t *testing. {Name: "main"}, }} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{ KubernetesExecutorConfig: &kubernetesplatform.KubernetesExecutorConfig{ @@ -2567,7 +2569,7 @@ func Test_extendPodSpecPatch_ImagePullPolicy(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), tt.podSpec, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -2765,7 +2767,7 @@ func Test_extendPodSpecPatch_GenericEphemeralVolume(t *testing.T) { taskConfig := &TaskConfig{} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), tt.podSpec, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -3069,7 +3071,7 @@ func Test_extendPodSpecPatch_NodeAffinity(t *testing.T) { taskConfig := &TaskConfig{} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), got, Options{KubernetesExecutorConfig: tt.k8sExecCfg}, nil, @@ -3154,7 +3156,7 @@ func Test_extendPodSpecPatch_TaskConfig_CapturesAndApplies(t *testing.T) { taskCfg := &TaskConfig{} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), podSpec, Options{KubernetesExecutorConfig: cfg, Component: comp}, nil, @@ -3305,7 +3307,7 @@ func Test_extendPodSpecPatch_PvcMounts_Passthrough_NotAppliedToPod(t *testing.T) } taskCfg := &TaskConfig{} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), podSpec, Options{KubernetesExecutorConfig: cfg, Component: comp}, nil, @@ -3339,7 +3341,7 @@ func Test_extendPodSpecPatch_PvcMounts_Passthrough_AppliedToPod(t *testing.T) { } taskCfg := &TaskConfig{} err := extendPodSpecPatch( - context.Background(), + util.WithExistingLogger(context.Background(), logrus.New()), podSpec, Options{KubernetesExecutorConfig: cfg, Component: comp}, nil, From 4754bf94e74633064aacc5c4ab9cb4ef74066a0f Mon Sep 17 00:00:00 2001 From: arpechenin Date: Tue, 10 Mar 2026 00:02:51 +0300 Subject: [PATCH 37/51] add additional rbac Signed-off-by: arpechenin --- .../pipelines-profile-controller-admin.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/pipelines-profile-controller-admin.yaml b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/pipelines-profile-controller-admin.yaml index 8286abc1d5f..c0cfd935df7 100644 --- a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/pipelines-profile-controller-admin.yaml +++ b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/pipelines-profile-controller-admin.yaml @@ -18,6 +18,9 @@ rules: - apiGroups: [ "" ] resources: [ "configmaps" ] verbs: [ "get", "list", "watch" ] + - apiGroups: [ "" ] + resources: [ "secrets" ] + verbs: [ "get", "list", "watch" ] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding From f4370537222329c1827bb9b9397de5e70fe266a4 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Tue, 10 Mar 2026 02:36:46 +0300 Subject: [PATCH 38/51] fix error message Signed-off-by: arpechenin --- backend/src/driver/rpc_handler.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/src/driver/rpc_handler.go b/backend/src/driver/rpc_handler.go index c44ebf9ac62..af80122e956 100644 --- a/backend/src/driver/rpc_handler.go +++ b/backend/src/driver/rpc_handler.go @@ -162,9 +162,9 @@ func drive(args api.DriverPluginArgs) (execution *driver.Execution, err error) { return nil, fmt.Errorf("failed to create driver logger: %v", err) } defer func() { - err = os.Remove(logFile) - if err != nil { - glog.Errorf("Failed to remove processed log file: %v", err) + removeErr := os.Remove(logFile) + if removeErr != nil { + glog.Errorf("Failed to remove processed log file: %v", removeErr) } }() defer func() { @@ -187,9 +187,9 @@ func drive(args api.DriverPluginArgs) (execution *driver.Execution, err error) { }() defer func() { if f != nil { - err = f.Close() - if err != nil { - glog.Errorf("Failed to close file: %v", err) + closeErr := f.Close() + if closeErr != nil { + glog.Errorf("Failed to close file: %v", closeErr) } } }() From f1bd7cc21325efe5d2cd89211fa0713bf83ae2be Mon Sep 17 00:00:00 2001 From: arpechenin Date: Tue, 10 Mar 2026 02:43:48 +0300 Subject: [PATCH 39/51] print events Signed-off-by: arpechenin --- .github/workflows/e2e-test.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index f280d56ba55..8fd55daab9a 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -160,6 +160,13 @@ jobs: tls_enabled: ${{ matrix.pod_to_pod_tls_enabled }} ca_cert_path: ${{ env.CA_CERT_PATH }} + - name: Print Argo WF Specs events + if: always() + run: | + kubectl get events -A \ + --field-selector involvedObject.kind=Workflow \ + --sort-by=.lastTimestamp + end-to-end-critical-scenario-multi-user-tests: runs-on: ubuntu-latest needs: build From 01b07550950e3af5d52f52d10e534b84354498f7 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Tue, 10 Mar 2026 10:10:48 +0300 Subject: [PATCH 40/51] remove debug logs Signed-off-by: arpechenin --- .github/workflows/e2e-test.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 8fd55daab9a..f280d56ba55 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -160,13 +160,6 @@ jobs: tls_enabled: ${{ matrix.pod_to_pod_tls_enabled }} ca_cert_path: ${{ env.CA_CERT_PATH }} - - name: Print Argo WF Specs events - if: always() - run: | - kubectl get events -A \ - --field-selector involvedObject.kind=Workflow \ - --sort-by=.lastTimestamp - end-to-end-critical-scenario-multi-user-tests: runs-on: ubuntu-latest needs: build From 93fabba46520a96b33dc2014331dc1ed10b8e89d Mon Sep 17 00:00:00 2001 From: arpechenin Date: Fri, 13 Mar 2026 13:51:29 +0300 Subject: [PATCH 41/51] Prevent log file race condition in for loop by adding per-request millisecond timestamp prefix Signed-off-by: arpechenin --- backend/src/driver/rpc_handler.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/src/driver/rpc_handler.go b/backend/src/driver/rpc_handler.go index af80122e956..88ed2e3250b 100644 --- a/backend/src/driver/rpc_handler.go +++ b/backend/src/driver/rpc_handler.go @@ -24,6 +24,7 @@ import ( "os" "strconv" "strings" + "time" "github.com/golang/glog" "github.com/google/uuid" @@ -154,7 +155,7 @@ func drive(args api.DriverPluginArgs) (execution *driver.Execution, err error) { outputPathPrefix string ) var pipeline *metadata.Pipeline - logID := fmt.Sprintf("%v-%v-%v", args.IterationIndex, args.Type, args.TaskName) + logID := fmt.Sprintf("%d-%v-%v-%v", time.Now().UnixMilli(), args.IterationIndex, args.Type, args.TaskName) logDir := "/kfp/log" logFile := fmt.Sprintf("%s/%s.log", logDir, logID) ctx, f, err := util.WithLogger(context.Background(), logFile) From 1716f328957c40b34f633351c762805d5f44ec2d Mon Sep 17 00:00:00 2001 From: arpechenin Date: Fri, 27 Mar 2026 21:44:20 +0300 Subject: [PATCH 42/51] Move driver logs code to React Query v5 Signed-off-by: arpechenin --- .../src/components/tabs/RuntimeNodeDetailsV2.tsx | 12 ++++++------ frontend/src/hooks/queryKeys.ts | 3 +++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/frontend/src/components/tabs/RuntimeNodeDetailsV2.tsx b/frontend/src/components/tabs/RuntimeNodeDetailsV2.tsx index 9b638f626d3..c1faf348d37 100644 --- a/frontend/src/components/tabs/RuntimeNodeDetailsV2.tsx +++ b/frontend/src/components/tabs/RuntimeNodeDetailsV2.tsx @@ -169,16 +169,16 @@ function TaskNodeDetail({ enabled: !!execution, }); - const { data: driverLogsInfo } = useQuery, Error>( - ['driver-logs', { executionId: execution?.getId(), namespace }], - async () => { + const { data: driverLogsInfo } = useQuery({ + queryKey: queryKeys.executionLogs(execution?.getId(), namespace), + queryFn: async (): Promise> => { if (!execution) { throw new Error('No execution is found.'); } - return await getDriverLogsInfo(execution, namespace); + return getDriverLogsInfo(execution, namespace); }, - { enabled: !!execution }, - ); + enabled: !!execution, + }); const logsDetails = logsInfo?.get(LOGS_DETAILS); const logsBannerMessage = logsInfo?.get(LOGS_BANNER_MESSAGE); diff --git a/frontend/src/hooks/queryKeys.ts b/frontend/src/hooks/queryKeys.ts index cc70d74effd..faa3a0bccb8 100644 --- a/frontend/src/hooks/queryKeys.ts +++ b/frontend/src/hooks/queryKeys.ts @@ -65,6 +65,9 @@ export const queryKeys = { executionLogs: (executionId: number | undefined, namespace: string | undefined) => ['execution_logs', { executionId, namespace }] as const, + driverLogs: (executionId: number | undefined, namespace: string | undefined) => + ['driver-logs', { executionId, namespace }] as const, + contextByExecution: (executionId: number, executionState: number) => ['context_by_execution', { id: executionId, state: executionState }] as const, From 1f023c5f4ab661fd70316e8e369d3cd815b29cd1 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Fri, 27 Mar 2026 22:09:26 +0300 Subject: [PATCH 43/51] fix linter errors Signed-off-by: arpechenin --- backend/src/v2/compiler/argocompiler/container_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/src/v2/compiler/argocompiler/container_test.go b/backend/src/v2/compiler/argocompiler/container_test.go index 1682874d790..db01e87c8cb 100644 --- a/backend/src/v2/compiler/argocompiler/container_test.go +++ b/backend/src/v2/compiler/argocompiler/container_test.go @@ -85,7 +85,8 @@ func TestContainerDriverTemplate_IncludesKFPPodNameEnv(t *testing.T) { job: &pipelinespec.PipelineJob{}, } - name := c.addContainerDriverTemplate() + name, err := c.addContainerDriverTemplate() + require.NoError(t, err) require.Equal(t, "system-container-driver", name) tmpl, exists := c.templates[name] From 209ecc75f98b1dc6d08ada80e03c43711f2201d1 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Fri, 27 Mar 2026 22:20:26 +0300 Subject: [PATCH 44/51] fix workflow-compiler tests after merge Signed-off-by: arpechenin --- test_data/compiled-workflows/add_numbers.yaml | 1 - .../arguments_parameters.yaml | 1 - .../compiled-workflows/artifact_cache.yaml | 1 - .../compiled-workflows/artifact_crust.yaml | 1 - .../compiled-workflows/artifacts_complex.yaml | 1 - .../compiled-workflows/artifacts_simple.yaml | 1 - .../collected_artifacts.yaml | 1 - .../collected_parameters.yaml | 1 - .../component_with_metadata_fields.yaml | 1 - .../component_with_optional_inputs.yaml | 1 - .../component_with_pip_index_urls.yaml | 1 - .../component_with_pip_install.yaml | 1 - .../component_with_pip_install_in_venv.yaml | 1 - .../components_with_optional_artifacts.yaml | 1 - .../compiled-workflows/concat_message.yaml | 1 - .../conditional_producer_and_consumers.yaml | 1 - .../container_component_with_no_inputs.yaml | 1 - .../compiled-workflows/container_io.yaml | 1 - .../container_no_input.yaml | 1 - .../container_with_artifact_output.yaml | 1 - .../container_with_concat_placeholder.yaml | 1 - .../container_with_if_placeholder.yaml | 1 - ...container_with_placeholder_in_fstring.yaml | 1 - .../containerized_python_component.yaml | 1 - .../create_pod_metadata_complex.yaml | 1 - .../cross_loop_after_topology.yaml | 1 - test_data/compiled-workflows/dict_input.yaml | 1 - .../compiled-workflows/embedded_artifact.yaml | 1 - test_data/compiled-workflows/env-var.yaml | 1 - test_data/compiled-workflows/fail_v2.yaml | 1 - test_data/compiled-workflows/flip_coin.yaml | 1 - test_data/compiled-workflows/hello_world.yaml | 1 - test_data/compiled-workflows/identity.yaml | 1 - .../if_elif_else_complex.yaml | 1 - .../if_elif_else_with_oneof_parameters.yaml | 1 - .../if_else_with_oneof_artifacts.yaml | 1 - .../if_else_with_oneof_parameters.yaml | 1 - .../compiled-workflows/input_artifact.yaml | 1 - .../iris_pipeline_compiled.yaml | 1 - ...lightweight_python_functions_pipeline.yaml | 1 - ...tweight_python_functions_with_outputs.yaml | 1 - .../log_streaming_compiled.yaml | 1 - .../compiled-workflows/long-running.yaml | 1 - .../loop_consume_upstream.yaml | 1 - .../metrics_visualization_v2.yaml | 1 - .../missing_kubernetes_optional_inputs.yaml | 1 - .../compiled-workflows/mixed_parameters.yaml | 1 - test_data/compiled-workflows/modelcar.yaml | 1 - .../mounted_cabundle_configmap.yaml | 1 - .../mounted_cabundle_secret.yaml | 1 - .../multiple_artifacts_namedtuple.yaml | 1 - .../multiple_parameters_namedtuple.yaml | 1 - ...peline_opt_input_child_level_compiled.yaml | 1 - ...sted_pipeline_opt_inputs_nil_compiled.yaml | 1 - ...line_opt_inputs_parent_level_compiled.yaml | 1 - .../compiled-workflows/nested_return.yaml | 1 - .../nested_with_parameters.yaml | 1 - .../notebook_component_mixed.yaml | 1 - .../notebook_component_simple.yaml | 1 - .../compiled-workflows/output_metrics.yaml | 1 - .../parallel_for_after_dependency.yaml | 1 - .../compiled-workflows/parameter_cache.yaml | 1 - .../compiled-workflows/parameter_oneof.yaml | 1 - .../parameters_complex.yaml | 1 - .../compiled-workflows/parameters_simple.yaml | 1 - .../pipeline_as_exit_task.yaml | 1 - .../pipeline_in_pipeline.yaml | 1 - .../pipeline_in_pipeline_complex.yaml | 1 - ...pipeline_in_pipeline_loaded_from_yaml.yaml | 1 - .../pipeline_producer_consumer.yaml | 1 - .../pipeline_with_after.yaml | 1 - .../pipeline_with_artifact_custom_path.yaml | 1 - ...ipeline_with_artifact_upload_download.yaml | 1 - .../pipeline_with_concat_placeholder.yaml | 1 - .../pipeline_with_condition.yaml | 1 - ...namic_task_output_custom_training_job.yaml | 1 - ...peline_with_dynamic_importer_metadata.yaml | 1 - ...namic_task_output_custom_training_job.yaml | 1 - .../compiled-workflows/pipeline_with_env.yaml | 1 - .../pipeline_with_exit_handler.yaml | 1 - .../pipeline_with_google_artifact_type.yaml | 1 - .../pipeline_with_importer.yaml | 1 - ...pipeline_with_importer_and_gcpc_types.yaml | 1 - .../pipeline_with_importer_workspace.yaml | 2 - .../pipeline_with_input_status_state.yaml | 1 - .../pipeline_with_loops.yaml | 1 - .../pipeline_with_loops_and_conditions.yaml | 1 - .../pipeline_with_metadata_fields.yaml | 1 - .../pipeline_with_metrics_outputs.yaml | 1 - .../pipeline_with_multiple_exit_handlers.yaml | 1 - .../pipeline_with_nested_conditions.yaml | 1 - .../pipeline_with_nested_conditions_yaml.yaml | 1 - .../pipeline_with_nested_loops.yaml | 1 - .../pipeline_with_only_display_name.yaml | 1 - .../pipeline_with_outputs.yaml | 1 - ...pipeline_with_parallelfor_parallelism.yaml | 1 - ...ipeline_with_params_containing_format.yaml | 1 - .../pipeline_with_placeholders.yaml | 1 - .../pipeline_with_pod_metadata.yaml | 1 - .../pipeline_with_retry.yaml | 1 - .../pipeline_with_reused_component.yaml | 1 - .../pipeline_with_secret_as_env.yaml | 1 - .../pipeline_with_secret_as_volume.yaml | 1 - ..._string_machine_fields_pipeline_input.yaml | 1 - ...ith_string_machine_fields_task_output.yaml | 9 +- .../pipeline_with_submit_request.yaml | 1 - .../pipeline_with_task_final_status.yaml | 1 - .../pipeline_with_task_final_status_yaml.yaml | 1 - ...th_task_using_ignore_upstream_failure.yaml | 1 - .../pipeline_with_utils.yaml | 1 - .../pipeline_with_various_io_types.yaml | 1 - .../pipeline_with_volume.yaml | 1 - .../pipeline_with_volume_long_name.yaml | 203 ++++++------------ .../pipeline_with_volume_no_cache.yaml | 1 - .../pipeline_with_workspace.yaml | 2 - ..._with_if_placeholder_none_input_value.yaml | 1 - test_data/compiled-workflows/preprocess.yaml | 1 - .../producer_consumer_param_pipeline.yaml | 1 - test_data/compiled-workflows/pvc_mount.yaml | 1 - .../pythonic_artifact_with_single_return.yaml | 1 - .../pythonic_artifacts_test_pipeline.yaml | 1 - ...onic_artifacts_with_list_of_artifacts.yaml | 1 - ...honic_artifacts_with_multiple_returns.yaml | 1 - .../ray_integration_compiled.yaml | 1 - .../run_as_user_cache_disabled.yaml | 1 - .../run_as_user_cache_enabled.yaml | 1 - .../compiled-workflows/sequential_v1.yaml | 1 - .../compiled-workflows/sequential_v2.yaml | 1 - .../compiled-workflows/take_nap_compiled.yaml | 1 - .../take_nap_pipeline_root_compiled.yaml | 1 - .../compiled-workflows/two_step_pipeline.yaml | 1 - .../two_step_pipeline_containerized.yaml | 1 - .../upload_download_compiled.yaml | 1 - .../xgboost_sample_pipeline.yaml | 1 - 134 files changed, 65 insertions(+), 281 deletions(-) diff --git a/test_data/compiled-workflows/add_numbers.yaml b/test_data/compiled-workflows/add_numbers.yaml index 26a9f2c0bb0..5d34296fa04 100644 --- a/test_data/compiled-workflows/add_numbers.yaml +++ b/test_data/compiled-workflows/add_numbers.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: add-numbers- spec: arguments: diff --git a/test_data/compiled-workflows/arguments_parameters.yaml b/test_data/compiled-workflows/arguments_parameters.yaml index 7d0b50ce3ad..6db4c0d4011 100644 --- a/test_data/compiled-workflows/arguments_parameters.yaml +++ b/test_data/compiled-workflows/arguments_parameters.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: echo- spec: arguments: diff --git a/test_data/compiled-workflows/artifact_cache.yaml b/test_data/compiled-workflows/artifact_cache.yaml index aa85fa4f3fa..1dd859cff31 100644 --- a/test_data/compiled-workflows/artifact_cache.yaml +++ b/test_data/compiled-workflows/artifact_cache.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: artifact-cache-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/artifact_crust.yaml b/test_data/compiled-workflows/artifact_crust.yaml index b4c4ba49809..f1a5fcc7324 100644 --- a/test_data/compiled-workflows/artifact_crust.yaml +++ b/test_data/compiled-workflows/artifact_crust.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: artifact-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/artifacts_complex.yaml b/test_data/compiled-workflows/artifacts_complex.yaml index 5bf2da7cab6..b09e7ae7229 100644 --- a/test_data/compiled-workflows/artifacts_complex.yaml +++ b/test_data/compiled-workflows/artifacts_complex.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: math-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/artifacts_simple.yaml b/test_data/compiled-workflows/artifacts_simple.yaml index b67bd02a457..bf86dae3fa1 100644 --- a/test_data/compiled-workflows/artifacts_simple.yaml +++ b/test_data/compiled-workflows/artifacts_simple.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: math-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/collected_artifacts.yaml b/test_data/compiled-workflows/collected_artifacts.yaml index af85c7e6605..cd3c934770a 100644 --- a/test_data/compiled-workflows/collected_artifacts.yaml +++ b/test_data/compiled-workflows/collected_artifacts.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: collected-artifact-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/collected_parameters.yaml b/test_data/compiled-workflows/collected_parameters.yaml index 73f5b682c80..4085d458374 100644 --- a/test_data/compiled-workflows/collected_parameters.yaml +++ b/test_data/compiled-workflows/collected_parameters.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: collected-param-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/component_with_metadata_fields.yaml b/test_data/compiled-workflows/component_with_metadata_fields.yaml index 18361094a95..02739f1945d 100644 --- a/test_data/compiled-workflows/component_with_metadata_fields.yaml +++ b/test_data/compiled-workflows/component_with_metadata_fields.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: dataset-joiner- spec: arguments: diff --git a/test_data/compiled-workflows/component_with_optional_inputs.yaml b/test_data/compiled-workflows/component_with_optional_inputs.yaml index 79bf10709d3..f66bf9ca47e 100644 --- a/test_data/compiled-workflows/component_with_optional_inputs.yaml +++ b/test_data/compiled-workflows/component_with_optional_inputs.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: v2-component-optional-input- spec: arguments: diff --git a/test_data/compiled-workflows/component_with_pip_index_urls.yaml b/test_data/compiled-workflows/component_with_pip_index_urls.yaml index 35a8514c3cd..2cc5d6c3654 100644 --- a/test_data/compiled-workflows/component_with_pip_index_urls.yaml +++ b/test_data/compiled-workflows/component_with_pip_index_urls.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: v2-component-pip-index-urls- spec: arguments: diff --git a/test_data/compiled-workflows/component_with_pip_install.yaml b/test_data/compiled-workflows/component_with_pip_install.yaml index d0c656781ef..b3498d61877 100644 --- a/test_data/compiled-workflows/component_with_pip_install.yaml +++ b/test_data/compiled-workflows/component_with_pip_install.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: component-with-pip-install- spec: arguments: diff --git a/test_data/compiled-workflows/component_with_pip_install_in_venv.yaml b/test_data/compiled-workflows/component_with_pip_install_in_venv.yaml index 34d38127f55..68c58e4c254 100644 --- a/test_data/compiled-workflows/component_with_pip_install_in_venv.yaml +++ b/test_data/compiled-workflows/component_with_pip_install_in_venv.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: component-with-pip-install- spec: arguments: diff --git a/test_data/compiled-workflows/components_with_optional_artifacts.yaml b/test_data/compiled-workflows/components_with_optional_artifacts.yaml index 8b9bce86ee3..6fc0c457cfd 100644 --- a/test_data/compiled-workflows/components_with_optional_artifacts.yaml +++ b/test_data/compiled-workflows/components_with_optional_artifacts.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: optional-artifact-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/concat_message.yaml b/test_data/compiled-workflows/concat_message.yaml index d0c99d5cec6..b7ac9d67574 100644 --- a/test_data/compiled-workflows/concat_message.yaml +++ b/test_data/compiled-workflows/concat_message.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: concat-message- spec: arguments: diff --git a/test_data/compiled-workflows/conditional_producer_and_consumers.yaml b/test_data/compiled-workflows/conditional_producer_and_consumers.yaml index 3d49caef58e..3adb435872b 100644 --- a/test_data/compiled-workflows/conditional_producer_and_consumers.yaml +++ b/test_data/compiled-workflows/conditional_producer_and_consumers.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: math-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/container_component_with_no_inputs.yaml b/test_data/compiled-workflows/container_component_with_no_inputs.yaml index d4bf7ff77ff..35bc6e1c73d 100644 --- a/test_data/compiled-workflows/container_component_with_no_inputs.yaml +++ b/test_data/compiled-workflows/container_component_with_no_inputs.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: v2-container-component-no-input- spec: arguments: diff --git a/test_data/compiled-workflows/container_io.yaml b/test_data/compiled-workflows/container_io.yaml index 6199429eee1..b3715ba6c27 100644 --- a/test_data/compiled-workflows/container_io.yaml +++ b/test_data/compiled-workflows/container_io.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: container-io- spec: arguments: diff --git a/test_data/compiled-workflows/container_no_input.yaml b/test_data/compiled-workflows/container_no_input.yaml index 961fff99b6a..1d3fb621416 100644 --- a/test_data/compiled-workflows/container_no_input.yaml +++ b/test_data/compiled-workflows/container_no_input.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: container-no-input- spec: arguments: diff --git a/test_data/compiled-workflows/container_with_artifact_output.yaml b/test_data/compiled-workflows/container_with_artifact_output.yaml index b233f0cf7b2..10b22c71a4d 100644 --- a/test_data/compiled-workflows/container_with_artifact_output.yaml +++ b/test_data/compiled-workflows/container_with_artifact_output.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: container-with-artifact-output- spec: arguments: diff --git a/test_data/compiled-workflows/container_with_concat_placeholder.yaml b/test_data/compiled-workflows/container_with_concat_placeholder.yaml index 9c337aff62d..d7f33ce00c6 100644 --- a/test_data/compiled-workflows/container_with_concat_placeholder.yaml +++ b/test_data/compiled-workflows/container_with_concat_placeholder.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: container-with-concat-placeholder- spec: arguments: diff --git a/test_data/compiled-workflows/container_with_if_placeholder.yaml b/test_data/compiled-workflows/container_with_if_placeholder.yaml index 92673a8f2de..beb895d6d15 100644 --- a/test_data/compiled-workflows/container_with_if_placeholder.yaml +++ b/test_data/compiled-workflows/container_with_if_placeholder.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: container-with-if-placeholder- spec: arguments: diff --git a/test_data/compiled-workflows/container_with_placeholder_in_fstring.yaml b/test_data/compiled-workflows/container_with_placeholder_in_fstring.yaml index 3e7aafb9bcc..9b399dc62b0 100644 --- a/test_data/compiled-workflows/container_with_placeholder_in_fstring.yaml +++ b/test_data/compiled-workflows/container_with_placeholder_in_fstring.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: container-with-placeholder-in-fstring- spec: arguments: diff --git a/test_data/compiled-workflows/containerized_python_component.yaml b/test_data/compiled-workflows/containerized_python_component.yaml index 2e0b2991bdf..60748357d9c 100644 --- a/test_data/compiled-workflows/containerized_python_component.yaml +++ b/test_data/compiled-workflows/containerized_python_component.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: concat-message- spec: arguments: diff --git a/test_data/compiled-workflows/create_pod_metadata_complex.yaml b/test_data/compiled-workflows/create_pod_metadata_complex.yaml index 6c8a9880ace..d0b9e3c58cb 100644 --- a/test_data/compiled-workflows/create_pod_metadata_complex.yaml +++ b/test_data/compiled-workflows/create_pod_metadata_complex.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-with-pod-metadata- spec: arguments: diff --git a/test_data/compiled-workflows/cross_loop_after_topology.yaml b/test_data/compiled-workflows/cross_loop_after_topology.yaml index eff852c4189..5990cc4ba25 100644 --- a/test_data/compiled-workflows/cross_loop_after_topology.yaml +++ b/test_data/compiled-workflows/cross_loop_after_topology.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: my-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/dict_input.yaml b/test_data/compiled-workflows/dict_input.yaml index 6f55b508c17..4925237e98b 100644 --- a/test_data/compiled-workflows/dict_input.yaml +++ b/test_data/compiled-workflows/dict_input.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: dict-input- spec: arguments: diff --git a/test_data/compiled-workflows/embedded_artifact.yaml b/test_data/compiled-workflows/embedded_artifact.yaml index c23bd70ad5b..e97e753891d 100644 --- a/test_data/compiled-workflows/embedded_artifact.yaml +++ b/test_data/compiled-workflows/embedded_artifact.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: nb-simple- spec: arguments: diff --git a/test_data/compiled-workflows/env-var.yaml b/test_data/compiled-workflows/env-var.yaml index c2100ee100a..dd8dc8c3599 100644 --- a/test_data/compiled-workflows/env-var.yaml +++ b/test_data/compiled-workflows/env-var.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: test-env-exists- spec: arguments: diff --git a/test_data/compiled-workflows/fail_v2.yaml b/test_data/compiled-workflows/fail_v2.yaml index ed7d088bd9c..9f14ccfd109 100644 --- a/test_data/compiled-workflows/fail_v2.yaml +++ b/test_data/compiled-workflows/fail_v2.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: fail-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/flip_coin.yaml b/test_data/compiled-workflows/flip_coin.yaml index e71e73096d4..8cdce9ac6c4 100644 --- a/test_data/compiled-workflows/flip_coin.yaml +++ b/test_data/compiled-workflows/flip_coin.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: conditional-execution-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/hello_world.yaml b/test_data/compiled-workflows/hello_world.yaml index 07c6f278353..3502e68221a 100644 --- a/test_data/compiled-workflows/hello_world.yaml +++ b/test_data/compiled-workflows/hello_world.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: echo- spec: arguments: diff --git a/test_data/compiled-workflows/identity.yaml b/test_data/compiled-workflows/identity.yaml index b59d60161d9..f33105f3a7e 100644 --- a/test_data/compiled-workflows/identity.yaml +++ b/test_data/compiled-workflows/identity.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: identity- spec: arguments: diff --git a/test_data/compiled-workflows/if_elif_else_complex.yaml b/test_data/compiled-workflows/if_elif_else_complex.yaml index 8b9a58684f3..2a4ba11edf8 100644 --- a/test_data/compiled-workflows/if_elif_else_complex.yaml +++ b/test_data/compiled-workflows/if_elif_else_complex.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: lucky-number-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/if_elif_else_with_oneof_parameters.yaml b/test_data/compiled-workflows/if_elif_else_with_oneof_parameters.yaml index 1e4fc2a7315..1c7a5d758c3 100644 --- a/test_data/compiled-workflows/if_elif_else_with_oneof_parameters.yaml +++ b/test_data/compiled-workflows/if_elif_else_with_oneof_parameters.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: outer-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/if_else_with_oneof_artifacts.yaml b/test_data/compiled-workflows/if_else_with_oneof_artifacts.yaml index 2aa5c9cfddc..01f1f75fc77 100644 --- a/test_data/compiled-workflows/if_else_with_oneof_artifacts.yaml +++ b/test_data/compiled-workflows/if_else_with_oneof_artifacts.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: outer-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/if_else_with_oneof_parameters.yaml b/test_data/compiled-workflows/if_else_with_oneof_parameters.yaml index 5a8ff930650..e43df4ecf76 100644 --- a/test_data/compiled-workflows/if_else_with_oneof_parameters.yaml +++ b/test_data/compiled-workflows/if_else_with_oneof_parameters.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: flip-coin-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/input_artifact.yaml b/test_data/compiled-workflows/input_artifact.yaml index 9fe95342e0f..4c9b87d38cb 100644 --- a/test_data/compiled-workflows/input_artifact.yaml +++ b/test_data/compiled-workflows/input_artifact.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: input-artifact- spec: arguments: diff --git a/test_data/compiled-workflows/iris_pipeline_compiled.yaml b/test_data/compiled-workflows/iris_pipeline_compiled.yaml index 611cdda9615..992c83c1bb5 100644 --- a/test_data/compiled-workflows/iris_pipeline_compiled.yaml +++ b/test_data/compiled-workflows/iris_pipeline_compiled.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: iris-training-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/lightweight_python_functions_pipeline.yaml b/test_data/compiled-workflows/lightweight_python_functions_pipeline.yaml index 543e077b70d..3b4fe7a3b8f 100644 --- a/test_data/compiled-workflows/lightweight_python_functions_pipeline.yaml +++ b/test_data/compiled-workflows/lightweight_python_functions_pipeline.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: my-test-pipeline-beta- spec: arguments: diff --git a/test_data/compiled-workflows/lightweight_python_functions_with_outputs.yaml b/test_data/compiled-workflows/lightweight_python_functions_with_outputs.yaml index a7864a30301..076a862d832 100644 --- a/test_data/compiled-workflows/lightweight_python_functions_with_outputs.yaml +++ b/test_data/compiled-workflows/lightweight_python_functions_with_outputs.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: functions-with-outputs- spec: arguments: diff --git a/test_data/compiled-workflows/log_streaming_compiled.yaml b/test_data/compiled-workflows/log_streaming_compiled.yaml index a17b7e28949..849494bafa7 100644 --- a/test_data/compiled-workflows/log_streaming_compiled.yaml +++ b/test_data/compiled-workflows/log_streaming_compiled.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: log-streaming-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/long-running.yaml b/test_data/compiled-workflows/long-running.yaml index bfaafaf758e..e9438c111fa 100644 --- a/test_data/compiled-workflows/long-running.yaml +++ b/test_data/compiled-workflows/long-running.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: wait-awhile- spec: arguments: diff --git a/test_data/compiled-workflows/loop_consume_upstream.yaml b/test_data/compiled-workflows/loop_consume_upstream.yaml index 689d2b81ede..dd7d3570d9f 100644 --- a/test_data/compiled-workflows/loop_consume_upstream.yaml +++ b/test_data/compiled-workflows/loop_consume_upstream.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: loop-consume-upstream- spec: arguments: diff --git a/test_data/compiled-workflows/metrics_visualization_v2.yaml b/test_data/compiled-workflows/metrics_visualization_v2.yaml index daa771afac9..484927110d0 100644 --- a/test_data/compiled-workflows/metrics_visualization_v2.yaml +++ b/test_data/compiled-workflows/metrics_visualization_v2.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: metrics-visualization-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/missing_kubernetes_optional_inputs.yaml b/test_data/compiled-workflows/missing_kubernetes_optional_inputs.yaml index 68a954d0461..27ed2b19e2a 100644 --- a/test_data/compiled-workflows/missing_kubernetes_optional_inputs.yaml +++ b/test_data/compiled-workflows/missing_kubernetes_optional_inputs.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: missing-kubernetes-optional-inputs-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/mixed_parameters.yaml b/test_data/compiled-workflows/mixed_parameters.yaml index 45dae11c9f3..58cfc0466c9 100644 --- a/test_data/compiled-workflows/mixed_parameters.yaml +++ b/test_data/compiled-workflows/mixed_parameters.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: mixed-parameters-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/modelcar.yaml b/test_data/compiled-workflows/modelcar.yaml index 9c97b0d4818..fbc0016095a 100644 --- a/test_data/compiled-workflows/modelcar.yaml +++ b/test_data/compiled-workflows/modelcar.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-with-modelcar-model- spec: arguments: diff --git a/test_data/compiled-workflows/mounted_cabundle_configmap.yaml b/test_data/compiled-workflows/mounted_cabundle_configmap.yaml index f22a920bad3..c6a234ba866 100644 --- a/test_data/compiled-workflows/mounted_cabundle_configmap.yaml +++ b/test_data/compiled-workflows/mounted_cabundle_configmap.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: echo- spec: arguments: diff --git a/test_data/compiled-workflows/mounted_cabundle_secret.yaml b/test_data/compiled-workflows/mounted_cabundle_secret.yaml index 6c9b8eb0721..f53f204269f 100644 --- a/test_data/compiled-workflows/mounted_cabundle_secret.yaml +++ b/test_data/compiled-workflows/mounted_cabundle_secret.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: echo- spec: arguments: diff --git a/test_data/compiled-workflows/multiple_artifacts_namedtuple.yaml b/test_data/compiled-workflows/multiple_artifacts_namedtuple.yaml index 8e162a95168..ecc98d5a967 100644 --- a/test_data/compiled-workflows/multiple_artifacts_namedtuple.yaml +++ b/test_data/compiled-workflows/multiple_artifacts_namedtuple.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: multiple-artifacts-namedtuple-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/multiple_parameters_namedtuple.yaml b/test_data/compiled-workflows/multiple_parameters_namedtuple.yaml index c46a1ee2577..c060ec3b59b 100644 --- a/test_data/compiled-workflows/multiple_parameters_namedtuple.yaml +++ b/test_data/compiled-workflows/multiple_parameters_namedtuple.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: multiple-parameters-namedtuple-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/nested_pipeline_opt_input_child_level_compiled.yaml b/test_data/compiled-workflows/nested_pipeline_opt_input_child_level_compiled.yaml index db766cfb3ea..8538292aa32 100644 --- a/test_data/compiled-workflows/nested_pipeline_opt_input_child_level_compiled.yaml +++ b/test_data/compiled-workflows/nested_pipeline_opt_input_child_level_compiled.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: nested-pipeline-opt-input-child-level- spec: arguments: diff --git a/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml b/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml index a02c398c1b2..be030912dc5 100644 --- a/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml +++ b/test_data/compiled-workflows/nested_pipeline_opt_inputs_nil_compiled.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: nested-pipeline-opt-inputs-nil- spec: arguments: diff --git a/test_data/compiled-workflows/nested_pipeline_opt_inputs_parent_level_compiled.yaml b/test_data/compiled-workflows/nested_pipeline_opt_inputs_parent_level_compiled.yaml index 4769693969e..29bcb0b8ee9 100644 --- a/test_data/compiled-workflows/nested_pipeline_opt_inputs_parent_level_compiled.yaml +++ b/test_data/compiled-workflows/nested_pipeline_opt_inputs_parent_level_compiled.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: nested-pipeline-opt-inputs-parent-level- spec: arguments: diff --git a/test_data/compiled-workflows/nested_return.yaml b/test_data/compiled-workflows/nested_return.yaml index 0e23e0ed896..22b37d12337 100644 --- a/test_data/compiled-workflows/nested_return.yaml +++ b/test_data/compiled-workflows/nested_return.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: nested-return- spec: arguments: diff --git a/test_data/compiled-workflows/nested_with_parameters.yaml b/test_data/compiled-workflows/nested_with_parameters.yaml index 38aa8a0be0a..fa57ffa6c0d 100644 --- a/test_data/compiled-workflows/nested_with_parameters.yaml +++ b/test_data/compiled-workflows/nested_with_parameters.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: math-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/notebook_component_mixed.yaml b/test_data/compiled-workflows/notebook_component_mixed.yaml index 16e4ef6c6a4..d8759f11275 100644 --- a/test_data/compiled-workflows/notebook_component_mixed.yaml +++ b/test_data/compiled-workflows/notebook_component_mixed.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: nb-mixed- spec: arguments: diff --git a/test_data/compiled-workflows/notebook_component_simple.yaml b/test_data/compiled-workflows/notebook_component_simple.yaml index 0772e13b1d5..7c83cc14f22 100644 --- a/test_data/compiled-workflows/notebook_component_simple.yaml +++ b/test_data/compiled-workflows/notebook_component_simple.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: nb-simple- spec: arguments: diff --git a/test_data/compiled-workflows/output_metrics.yaml b/test_data/compiled-workflows/output_metrics.yaml index f9dbd7bc154..60f741089ec 100644 --- a/test_data/compiled-workflows/output_metrics.yaml +++ b/test_data/compiled-workflows/output_metrics.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: output-metrics- spec: arguments: diff --git a/test_data/compiled-workflows/parallel_for_after_dependency.yaml b/test_data/compiled-workflows/parallel_for_after_dependency.yaml index eb07ad6172a..d6274d5ecc5 100644 --- a/test_data/compiled-workflows/parallel_for_after_dependency.yaml +++ b/test_data/compiled-workflows/parallel_for_after_dependency.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: loop-with-after-dependency-set- spec: arguments: diff --git a/test_data/compiled-workflows/parameter_cache.yaml b/test_data/compiled-workflows/parameter_cache.yaml index 13d556aeb1f..284ea157c64 100644 --- a/test_data/compiled-workflows/parameter_cache.yaml +++ b/test_data/compiled-workflows/parameter_cache.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: parameter-cache-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/parameter_oneof.yaml b/test_data/compiled-workflows/parameter_oneof.yaml index a77d471d93e..8d124b3a3cf 100644 --- a/test_data/compiled-workflows/parameter_oneof.yaml +++ b/test_data/compiled-workflows/parameter_oneof.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: parameter-oneof-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/parameters_complex.yaml b/test_data/compiled-workflows/parameters_complex.yaml index d8d1fa9ea3b..2efe2806b18 100644 --- a/test_data/compiled-workflows/parameters_complex.yaml +++ b/test_data/compiled-workflows/parameters_complex.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: math-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/parameters_simple.yaml b/test_data/compiled-workflows/parameters_simple.yaml index 63cbe75e9a9..8ea6c6b6779 100644 --- a/test_data/compiled-workflows/parameters_simple.yaml +++ b/test_data/compiled-workflows/parameters_simple.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: math-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_as_exit_task.yaml b/test_data/compiled-workflows/pipeline_as_exit_task.yaml index dc05bdfc6b7..45ff561e27f 100644 --- a/test_data/compiled-workflows/pipeline_as_exit_task.yaml +++ b/test_data/compiled-workflows/pipeline_as_exit_task.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-with-task-final-status-conditional- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_in_pipeline.yaml b/test_data/compiled-workflows/pipeline_in_pipeline.yaml index 025c22292e5..08c27bc0697 100644 --- a/test_data/compiled-workflows/pipeline_in_pipeline.yaml +++ b/test_data/compiled-workflows/pipeline_in_pipeline.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-in-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml b/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml index 9c87c807b08..28091b220fe 100644 --- a/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml +++ b/test_data/compiled-workflows/pipeline_in_pipeline_complex.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-in-pipeline-complex- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_in_pipeline_loaded_from_yaml.yaml b/test_data/compiled-workflows/pipeline_in_pipeline_loaded_from_yaml.yaml index 4c06eb74de9..a62d275300a 100644 --- a/test_data/compiled-workflows/pipeline_in_pipeline_loaded_from_yaml.yaml +++ b/test_data/compiled-workflows/pipeline_in_pipeline_loaded_from_yaml.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-in-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_producer_consumer.yaml b/test_data/compiled-workflows/pipeline_producer_consumer.yaml index 96968191410..5b2381f6e5d 100644 --- a/test_data/compiled-workflows/pipeline_producer_consumer.yaml +++ b/test_data/compiled-workflows/pipeline_producer_consumer.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: math-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_after.yaml b/test_data/compiled-workflows/pipeline_with_after.yaml index 82e697785c1..96d67c6a1bd 100644 --- a/test_data/compiled-workflows/pipeline_with_after.yaml +++ b/test_data/compiled-workflows/pipeline_with_after.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-with-after- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_artifact_custom_path.yaml b/test_data/compiled-workflows/pipeline_with_artifact_custom_path.yaml index 6d3fce0d341..da2c5c12e98 100644 --- a/test_data/compiled-workflows/pipeline_with_artifact_custom_path.yaml +++ b/test_data/compiled-workflows/pipeline_with_artifact_custom_path.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-with-custom-path-artifact- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_artifact_upload_download.yaml b/test_data/compiled-workflows/pipeline_with_artifact_upload_download.yaml index b2a91f9dd6c..eb0ba8ff082 100644 --- a/test_data/compiled-workflows/pipeline_with_artifact_upload_download.yaml +++ b/test_data/compiled-workflows/pipeline_with_artifact_upload_download.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-with-datasets- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_concat_placeholder.yaml b/test_data/compiled-workflows/pipeline_with_concat_placeholder.yaml index 22ae7858cb4..924b3a09f9f 100644 --- a/test_data/compiled-workflows/pipeline_with_concat_placeholder.yaml +++ b/test_data/compiled-workflows/pipeline_with_concat_placeholder.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: one-step-pipeline-with-concat-placeholder- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_condition.yaml b/test_data/compiled-workflows/pipeline_with_condition.yaml index e06c6fef231..2913b43493b 100644 --- a/test_data/compiled-workflows/pipeline_with_condition.yaml +++ b/test_data/compiled-workflows/pipeline_with_condition.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: single-condition-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml b/test_data/compiled-workflows/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml index 989290e9ce2..96e732bbbe6 100644 --- a/test_data/compiled-workflows/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml +++ b/test_data/compiled-workflows/pipeline_with_condition_dynamic_task_output_custom_training_job.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-with-dynamic-condition-output- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_dynamic_importer_metadata.yaml b/test_data/compiled-workflows/pipeline_with_dynamic_importer_metadata.yaml index 800661365f2..88e2303000b 100644 --- a/test_data/compiled-workflows/pipeline_with_dynamic_importer_metadata.yaml +++ b/test_data/compiled-workflows/pipeline_with_dynamic_importer_metadata.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-with-importer- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_dynamic_task_output_custom_training_job.yaml b/test_data/compiled-workflows/pipeline_with_dynamic_task_output_custom_training_job.yaml index ca70715b017..5d074fe3ebe 100644 --- a/test_data/compiled-workflows/pipeline_with_dynamic_task_output_custom_training_job.yaml +++ b/test_data/compiled-workflows/pipeline_with_dynamic_task_output_custom_training_job.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_env.yaml b/test_data/compiled-workflows/pipeline_with_env.yaml index 4d3b51c6ac3..ad950333d53 100644 --- a/test_data/compiled-workflows/pipeline_with_env.yaml +++ b/test_data/compiled-workflows/pipeline_with_env.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-with-env- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_exit_handler.yaml b/test_data/compiled-workflows/pipeline_with_exit_handler.yaml index 5cfebf6fdae..9c252ab9c2b 100644 --- a/test_data/compiled-workflows/pipeline_with_exit_handler.yaml +++ b/test_data/compiled-workflows/pipeline_with_exit_handler.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-with-exit-handler- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_google_artifact_type.yaml b/test_data/compiled-workflows/pipeline_with_google_artifact_type.yaml index b366961be53..580ad5bf1fb 100644 --- a/test_data/compiled-workflows/pipeline_with_google_artifact_type.yaml +++ b/test_data/compiled-workflows/pipeline_with_google_artifact_type.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-with-google-types- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_importer.yaml b/test_data/compiled-workflows/pipeline_with_importer.yaml index 79d2c3afa58..0897496bd67 100644 --- a/test_data/compiled-workflows/pipeline_with_importer.yaml +++ b/test_data/compiled-workflows/pipeline_with_importer.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-with-importer- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_importer_and_gcpc_types.yaml b/test_data/compiled-workflows/pipeline_with_importer_and_gcpc_types.yaml index 94c4f279f2c..a41090eda0a 100644 --- a/test_data/compiled-workflows/pipeline_with_importer_and_gcpc_types.yaml +++ b/test_data/compiled-workflows/pipeline_with_importer_and_gcpc_types.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-with-importer-and-gcpc-type- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_importer_workspace.yaml b/test_data/compiled-workflows/pipeline_with_importer_workspace.yaml index 89df00e3c37..ec8092f9867 100644 --- a/test_data/compiled-workflows/pipeline_with_importer_workspace.yaml +++ b/test_data/compiled-workflows/pipeline_with_importer_workspace.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-with-importer-workspace- spec: arguments: @@ -659,7 +658,6 @@ spec: outputs: {} volumeClaimTemplates: - metadata: - creationTimestamp: null name: kfp-workspace spec: accessModes: diff --git a/test_data/compiled-workflows/pipeline_with_input_status_state.yaml b/test_data/compiled-workflows/pipeline_with_input_status_state.yaml index c9198cc09fc..1eda066bc84 100644 --- a/test_data/compiled-workflows/pipeline_with_input_status_state.yaml +++ b/test_data/compiled-workflows/pipeline_with_input_status_state.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: status-state-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_loops.yaml b/test_data/compiled-workflows/pipeline_with_loops.yaml index adf9ab8a685..5723013d180 100644 --- a/test_data/compiled-workflows/pipeline_with_loops.yaml +++ b/test_data/compiled-workflows/pipeline_with_loops.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-with-loops- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_loops_and_conditions.yaml b/test_data/compiled-workflows/pipeline_with_loops_and_conditions.yaml index 63d30ff1735..4d17b875066 100644 --- a/test_data/compiled-workflows/pipeline_with_loops_and_conditions.yaml +++ b/test_data/compiled-workflows/pipeline_with_loops_and_conditions.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-with-loops-and-conditions-multi-layers- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_metadata_fields.yaml b/test_data/compiled-workflows/pipeline_with_metadata_fields.yaml index ca2dead409e..e60222630ea 100644 --- a/test_data/compiled-workflows/pipeline_with_metadata_fields.yaml +++ b/test_data/compiled-workflows/pipeline_with_metadata_fields.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: dataset-concatenator- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_metrics_outputs.yaml b/test_data/compiled-workflows/pipeline_with_metrics_outputs.yaml index e202fabc5c3..6bea0523463 100644 --- a/test_data/compiled-workflows/pipeline_with_metrics_outputs.yaml +++ b/test_data/compiled-workflows/pipeline_with_metrics_outputs.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-with-metrics-outputs- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_multiple_exit_handlers.yaml b/test_data/compiled-workflows/pipeline_with_multiple_exit_handlers.yaml index cbed20cb632..b9de26b2894 100644 --- a/test_data/compiled-workflows/pipeline_with_multiple_exit_handlers.yaml +++ b/test_data/compiled-workflows/pipeline_with_multiple_exit_handlers.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-with-multiple-exit-handlers- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_nested_conditions.yaml b/test_data/compiled-workflows/pipeline_with_nested_conditions.yaml index 60caddfa9de..98383b9559d 100644 --- a/test_data/compiled-workflows/pipeline_with_nested_conditions.yaml +++ b/test_data/compiled-workflows/pipeline_with_nested_conditions.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: nested-conditions-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_nested_conditions_yaml.yaml b/test_data/compiled-workflows/pipeline_with_nested_conditions_yaml.yaml index 1162107adbf..48405fa8e60 100644 --- a/test_data/compiled-workflows/pipeline_with_nested_conditions_yaml.yaml +++ b/test_data/compiled-workflows/pipeline_with_nested_conditions_yaml.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: conditional-execution-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_nested_loops.yaml b/test_data/compiled-workflows/pipeline_with_nested_loops.yaml index 09292a9d078..0b6b88a027a 100644 --- a/test_data/compiled-workflows/pipeline_with_nested_loops.yaml +++ b/test_data/compiled-workflows/pipeline_with_nested_loops.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-with-nested-loops- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_only_display_name.yaml b/test_data/compiled-workflows/pipeline_with_only_display_name.yaml index 6f18284804d..7902ab40874 100644 --- a/test_data/compiled-workflows/pipeline_with_only_display_name.yaml +++ b/test_data/compiled-workflows/pipeline_with_only_display_name.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: echo-name- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_outputs.yaml b/test_data/compiled-workflows/pipeline_with_outputs.yaml index d1c175a3f22..2b31c3977e1 100644 --- a/test_data/compiled-workflows/pipeline_with_outputs.yaml +++ b/test_data/compiled-workflows/pipeline_with_outputs.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-in-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_parallelfor_parallelism.yaml b/test_data/compiled-workflows/pipeline_with_parallelfor_parallelism.yaml index 4cda2cd545f..fe1a6a41d08 100644 --- a/test_data/compiled-workflows/pipeline_with_parallelfor_parallelism.yaml +++ b/test_data/compiled-workflows/pipeline_with_parallelfor_parallelism.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-with-loops- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_params_containing_format.yaml b/test_data/compiled-workflows/pipeline_with_params_containing_format.yaml index 9cfa82134cc..0ae67544c43 100644 --- a/test_data/compiled-workflows/pipeline_with_params_containing_format.yaml +++ b/test_data/compiled-workflows/pipeline_with_params_containing_format.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-with-pipelineparam-containing-format- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_placeholders.yaml b/test_data/compiled-workflows/pipeline_with_placeholders.yaml index 6e7e89c45c3..72e86209e7a 100644 --- a/test_data/compiled-workflows/pipeline_with_placeholders.yaml +++ b/test_data/compiled-workflows/pipeline_with_placeholders.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-with-placeholders- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_pod_metadata.yaml b/test_data/compiled-workflows/pipeline_with_pod_metadata.yaml index 70e1498a6af..5942b69ef31 100644 --- a/test_data/compiled-workflows/pipeline_with_pod_metadata.yaml +++ b/test_data/compiled-workflows/pipeline_with_pod_metadata.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-with-pod-metadata- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_retry.yaml b/test_data/compiled-workflows/pipeline_with_retry.yaml index f5d51f8f703..2e9c06a6e72 100644 --- a/test_data/compiled-workflows/pipeline_with_retry.yaml +++ b/test_data/compiled-workflows/pipeline_with_retry.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: test-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_reused_component.yaml b/test_data/compiled-workflows/pipeline_with_reused_component.yaml index b5b85eca91e..dfe47459ac1 100644 --- a/test_data/compiled-workflows/pipeline_with_reused_component.yaml +++ b/test_data/compiled-workflows/pipeline_with_reused_component.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-with-reused-component- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml b/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml index 10163ffbcd4..1310d4e33e9 100644 --- a/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml +++ b/test_data/compiled-workflows/pipeline_with_secret_as_env.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-secret-env- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_secret_as_volume.yaml b/test_data/compiled-workflows/pipeline_with_secret_as_volume.yaml index c571d987e8a..6b2361d7c10 100644 --- a/test_data/compiled-workflows/pipeline_with_secret_as_volume.yaml +++ b/test_data/compiled-workflows/pipeline_with_secret_as_volume.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-secret-volume- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_string_machine_fields_pipeline_input.yaml b/test_data/compiled-workflows/pipeline_with_string_machine_fields_pipeline_input.yaml index dd949a8af53..9c753a0f3a1 100644 --- a/test_data/compiled-workflows/pipeline_with_string_machine_fields_pipeline_input.yaml +++ b/test_data/compiled-workflows/pipeline_with_string_machine_fields_pipeline_input.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml b/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml index 4a58110eef5..25eb6bd49da 100644 --- a/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml +++ b/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline- spec: arguments: @@ -72,7 +71,7 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef sum_numbers(a: int, b: int) -\u003e int:\n return a + b\n\n"],"image":"python:3.11","resources":{"accelerator":{"resourceCount":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}","resourceType":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"},"resourceCpuLimit":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}","resourceMemoryLimit":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}}' - name: components-root - value: '{"dag":{"tasks":{"accelerator-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-limit"},"taskInfo":{"name":"accelerator-limit"}},"accelerator-type":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-type"},"taskInfo":{"name":"accelerator-type"}},"cpu-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-cpu-limit"},"taskInfo":{"name":"cpu-limit"}},"memory-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-memory-limit"},"taskInfo":{"name":"memory-limit"}},"sum-numbers":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-type","memory-limit","accelerator-limit","cpu-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}}}}' + value: '{"dag":{"tasks":{"accelerator-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-limit"},"taskInfo":{"name":"accelerator-limit"}},"accelerator-type":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-type"},"taskInfo":{"name":"accelerator-type"}},"cpu-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-cpu-limit"},"taskInfo":{"name":"cpu-limit"}},"memory-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-memory-limit"},"taskInfo":{"name":"memory-limit"}},"sum-numbers":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-limit","accelerator-type","cpu-limit","memory-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}}}}' entrypoint: entrypoint podMetadata: annotations: @@ -357,15 +356,15 @@ spec: - name: component value: '{{workflow.parameters.components-49f9a898b718a077f30b7fd8c02d39767cff91ff0bbda4379daf866a91dbdb1b}}' - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-type","memory-limit","accelerator-limit","cpu-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}' + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-limit","accelerator-type","cpu-limit","memory-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}' - name: container value: '{{workflow.parameters.implementations-49f9a898b718a077f30b7fd8c02d39767cff91ff0bbda4379daf866a91dbdb1b}}' - name: task-name value: sum-numbers - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - depends: accelerator-type.Succeeded && memory-limit.Succeeded && accelerator-limit.Succeeded - && cpu-limit.Succeeded + depends: accelerator-limit.Succeeded && accelerator-type.Succeeded && cpu-limit.Succeeded + && memory-limit.Succeeded name: sum-numbers-driver template: system-container-driver - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_submit_request.yaml b/test_data/compiled-workflows/pipeline_with_submit_request.yaml index 9f22537055d..a995fd221ad 100644 --- a/test_data/compiled-workflows/pipeline_with_submit_request.yaml +++ b/test_data/compiled-workflows/pipeline_with_submit_request.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-with-external-request- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_task_final_status.yaml b/test_data/compiled-workflows/pipeline_with_task_final_status.yaml index 2a3d98ef662..a7a0cba2f99 100644 --- a/test_data/compiled-workflows/pipeline_with_task_final_status.yaml +++ b/test_data/compiled-workflows/pipeline_with_task_final_status.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-with-task-final-status- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_task_final_status_yaml.yaml b/test_data/compiled-workflows/pipeline_with_task_final_status_yaml.yaml index 617a4b0a10c..2a2d31df2bf 100644 --- a/test_data/compiled-workflows/pipeline_with_task_final_status_yaml.yaml +++ b/test_data/compiled-workflows/pipeline_with_task_final_status_yaml.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-with-task-final-status-yaml- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_task_using_ignore_upstream_failure.yaml b/test_data/compiled-workflows/pipeline_with_task_using_ignore_upstream_failure.yaml index fabbaf644a2..0dfa33e2aa8 100644 --- a/test_data/compiled-workflows/pipeline_with_task_using_ignore_upstream_failure.yaml +++ b/test_data/compiled-workflows/pipeline_with_task_using_ignore_upstream_failure.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: my-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_utils.yaml b/test_data/compiled-workflows/pipeline_with_utils.yaml index 229179e7651..4daffd32e71 100644 --- a/test_data/compiled-workflows/pipeline_with_utils.yaml +++ b/test_data/compiled-workflows/pipeline_with_utils.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-with-utils- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_various_io_types.yaml b/test_data/compiled-workflows/pipeline_with_various_io_types.yaml index 27dce2d5bfd..d5cb46ed3ef 100644 --- a/test_data/compiled-workflows/pipeline_with_various_io_types.yaml +++ b/test_data/compiled-workflows/pipeline_with_various_io_types.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-with-various-types- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_volume.yaml b/test_data/compiled-workflows/pipeline_with_volume.yaml index 3bbe1cce98e..fde538ee630 100644 --- a/test_data/compiled-workflows/pipeline_with_volume.yaml +++ b/test_data/compiled-workflows/pipeline_with_volume.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-with-volume- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_volume_long_name.yaml b/test_data/compiled-workflows/pipeline_with_volume_long_name.yaml index 3aa856343a9..3a5cd7a444f 100644 --- a/test_data/compiled-workflows/pipeline_with_volume_long_name.yaml +++ b/test_data/compiled-workflows/pipeline_with_volume_long_name.yaml @@ -46,6 +46,7 @@ spec: podMetadata: annotations: pipelines.kubeflow.org/v2_component: "true" + sidecar.istio.io/inject: "false" labels: pipelines.kubeflow.org/v2_component: "true" securityContext: @@ -54,81 +55,7 @@ spec: type: RuntimeDefault serviceAccountName: pipeline-runner templates: - - container: - args: - - --type - - CONTAINER - - --pipeline_name - - here-is-a-pipeline-very-long-name - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --container - - '{{inputs.parameters.container}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --cached_decision_path - - '{{outputs.parameters.cached-decision.path}}' - - --pod_spec_patch_path - - '{{outputs.parameters.pod-spec-patch.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --kubernetes_config - - '{{inputs.parameters.kubernetes-config}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - name: task @@ -146,16 +73,44 @@ spec: - name: pod-spec-patch valueFrom: default: "" - path: /tmp/outputs/pod-spec-patch + jsonPath: $.pod-spec-patch - default: "false" name: cached-decision valueFrom: default: "false" - path: /tmp/outputs/cached-decision + jsonPath: $.cached-decision - name: condition valueFrom: default: "true" - path: /tmp/outputs/condition + jsonPath: $.condition + plugin: + driver-plugin: + args: + cache_disabled: false + cached_decision_path: '{{outputs.parameters.cached-decision.path}}' + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + container: '{{inputs.parameters.container}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + http_proxy: "" + https_proxy: "" + iteration_index: '{{inputs.parameters.iteration-index}}' + kubernetes_config: '{{inputs.parameters.kubernetes-config}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: here-is-a-pipeline-very-long-name + pod_spec_patch_path: '{{outputs.parameters.pod-spec-patch.path}}' + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: CONTAINER securityContext: runAsNonRoot: true seccompProfile: @@ -314,70 +269,7 @@ spec: metadata: {} name: root outputs: {} - - container: - args: - - --type - - '{{inputs.parameters.driver-type}}' - - --pipeline_name - - here-is-a-pipeline-very-long-name - - --run_id - - '{{workflow.uid}}' - - --run_name - - '{{workflow.name}}' - - --run_display_name - - "" - - --dag_execution_id - - '{{inputs.parameters.parent-dag-id}}' - - --component - - '{{inputs.parameters.component}}' - - --task - - '{{inputs.parameters.task}}' - - --task_name - - '{{inputs.parameters.task-name}}' - - --runtime_config - - '{{inputs.parameters.runtime-config}}' - - --iteration_index - - '{{inputs.parameters.iteration-index}}' - - --execution_id_path - - '{{outputs.parameters.execution-id.path}}' - - --iteration_count_path - - '{{outputs.parameters.iteration-count.path}}' - - --condition_path - - '{{outputs.parameters.condition.path}}' - - --http_proxy - - "" - - --https_proxy - - "" - - --no_proxy - - "" - - --ml_pipeline_server_address - - ml-pipeline.kubeflow.svc.cluster.local - - --ml_pipeline_server_port - - "8887" - - --mlmd_server_address - - metadata-grpc-service.kubeflow.svc.cluster.local - - --mlmd_server_port - - "8080" - command: - - driver - image: ghcr.io/kubeflow/kfp-driver:latest - name: "" - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - inputs: + - inputs: parameters: - name: component - default: "" @@ -407,6 +299,33 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition + plugin: + driver-plugin: + args: + cache_disabled: false + component: '{{inputs.parameters.component}}' + condition_path: '{{outputs.parameters.condition.path}}' + dag_execution_id: '{{inputs.parameters.parent-dag-id}}' + execution_id_path: '{{outputs.parameters.execution-id.path}}' + http_proxy: "" + https_proxy: "" + iteration_count_path: '{{outputs.parameters.iteration-count.path}}' + iteration_index: '{{inputs.parameters.iteration-index}}' + metadata_tls_enabled: false + ml_pipeline_server_address: ml-pipeline.kubeflow.svc.cluster.local + ml_pipeline_server_port: "8887" + ml_pipeline_tls_enabled: false + mlmd_server_address: metadata-grpc-service.kubeflow.svc.cluster.local + mlmd_server_port: "8080" + no_proxy: "" + pipeline_name: here-is-a-pipeline-very-long-name + run_display_name: "" + run_id: '{{workflow.uid}}' + run_name: '{{workflow.name}}' + runtime_config: '{{inputs.parameters.runtime-config}}' + task: '{{inputs.parameters.task}}' + task_name: '{{inputs.parameters.task-name}}' + type: '{{inputs.parameters.driver-type}}' securityContext: runAsNonRoot: true seccompProfile: diff --git a/test_data/compiled-workflows/pipeline_with_volume_no_cache.yaml b/test_data/compiled-workflows/pipeline_with_volume_no_cache.yaml index fc9492b932c..cc22823b17a 100644 --- a/test_data/compiled-workflows/pipeline_with_volume_no_cache.yaml +++ b/test_data/compiled-workflows/pipeline_with_volume_no_cache.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-with-volume-no-cache- spec: arguments: diff --git a/test_data/compiled-workflows/pipeline_with_workspace.yaml b/test_data/compiled-workflows/pipeline_with_workspace.yaml index 0c12a46ae0f..9de15f929b5 100644 --- a/test_data/compiled-workflows/pipeline_with_workspace.yaml +++ b/test_data/compiled-workflows/pipeline_with_workspace.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pipeline-with-workspace- spec: arguments: @@ -371,7 +370,6 @@ spec: outputs: {} volumeClaimTemplates: - metadata: - creationTimestamp: null name: kfp-workspace spec: accessModes: diff --git a/test_data/compiled-workflows/placeholder_with_if_placeholder_none_input_value.yaml b/test_data/compiled-workflows/placeholder_with_if_placeholder_none_input_value.yaml index f0df5106f0a..4dc78bad584 100644 --- a/test_data/compiled-workflows/placeholder_with_if_placeholder_none_input_value.yaml +++ b/test_data/compiled-workflows/placeholder_with_if_placeholder_none_input_value.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: one-step-pipeline-with-if-placeholder-supply-none- spec: arguments: diff --git a/test_data/compiled-workflows/preprocess.yaml b/test_data/compiled-workflows/preprocess.yaml index 833d9ece30b..a624bc5a52b 100644 --- a/test_data/compiled-workflows/preprocess.yaml +++ b/test_data/compiled-workflows/preprocess.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: preprocess- spec: arguments: diff --git a/test_data/compiled-workflows/producer_consumer_param_pipeline.yaml b/test_data/compiled-workflows/producer_consumer_param_pipeline.yaml index 6ebcfb8baa6..ba7198fb3b5 100644 --- a/test_data/compiled-workflows/producer_consumer_param_pipeline.yaml +++ b/test_data/compiled-workflows/producer_consumer_param_pipeline.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: producer-consumer-param-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/pvc_mount.yaml b/test_data/compiled-workflows/pvc_mount.yaml index 63438a876da..f1c50f405d7 100644 --- a/test_data/compiled-workflows/pvc_mount.yaml +++ b/test_data/compiled-workflows/pvc_mount.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pvc-mount-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml b/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml index 77ae4745ea9..973a7b219e4 100644 --- a/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml +++ b/test_data/compiled-workflows/pythonic_artifact_with_single_return.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: make-language-model-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/pythonic_artifacts_test_pipeline.yaml b/test_data/compiled-workflows/pythonic_artifacts_test_pipeline.yaml index 160729f5014..757bff26884 100644 --- a/test_data/compiled-workflows/pythonic_artifacts_test_pipeline.yaml +++ b/test_data/compiled-workflows/pythonic_artifacts_test_pipeline.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: pythonic-artifacts-test- spec: arguments: diff --git a/test_data/compiled-workflows/pythonic_artifacts_with_list_of_artifacts.yaml b/test_data/compiled-workflows/pythonic_artifacts_with_list_of_artifacts.yaml index 7baaa5c1106..ac6a7e4c29c 100644 --- a/test_data/compiled-workflows/pythonic_artifacts_with_list_of_artifacts.yaml +++ b/test_data/compiled-workflows/pythonic_artifacts_with_list_of_artifacts.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: make-and-join-datasets- spec: arguments: diff --git a/test_data/compiled-workflows/pythonic_artifacts_with_multiple_returns.yaml b/test_data/compiled-workflows/pythonic_artifacts_with_multiple_returns.yaml index 961dffa401c..9a46c8aa408 100644 --- a/test_data/compiled-workflows/pythonic_artifacts_with_multiple_returns.yaml +++ b/test_data/compiled-workflows/pythonic_artifacts_with_multiple_returns.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: split-datasets-and-return-first- spec: arguments: diff --git a/test_data/compiled-workflows/ray_integration_compiled.yaml b/test_data/compiled-workflows/ray_integration_compiled.yaml index f223802e41e..4f5e0df8877 100644 --- a/test_data/compiled-workflows/ray_integration_compiled.yaml +++ b/test_data/compiled-workflows/ray_integration_compiled.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: ray-integration-test- spec: arguments: diff --git a/test_data/compiled-workflows/run_as_user_cache_disabled.yaml b/test_data/compiled-workflows/run_as_user_cache_disabled.yaml index 710f2d1af0e..3631b46bb1c 100644 --- a/test_data/compiled-workflows/run_as_user_cache_disabled.yaml +++ b/test_data/compiled-workflows/run_as_user_cache_disabled.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: echo- spec: arguments: diff --git a/test_data/compiled-workflows/run_as_user_cache_enabled.yaml b/test_data/compiled-workflows/run_as_user_cache_enabled.yaml index 710f2d1af0e..3631b46bb1c 100644 --- a/test_data/compiled-workflows/run_as_user_cache_enabled.yaml +++ b/test_data/compiled-workflows/run_as_user_cache_enabled.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: echo- spec: arguments: diff --git a/test_data/compiled-workflows/sequential_v1.yaml b/test_data/compiled-workflows/sequential_v1.yaml index 121f9e3ad6e..564d71d6b98 100644 --- a/test_data/compiled-workflows/sequential_v1.yaml +++ b/test_data/compiled-workflows/sequential_v1.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: sequential- spec: arguments: diff --git a/test_data/compiled-workflows/sequential_v2.yaml b/test_data/compiled-workflows/sequential_v2.yaml index 6f5db7ce06f..55cb33a69f8 100644 --- a/test_data/compiled-workflows/sequential_v2.yaml +++ b/test_data/compiled-workflows/sequential_v2.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: sequential- spec: arguments: diff --git a/test_data/compiled-workflows/take_nap_compiled.yaml b/test_data/compiled-workflows/take_nap_compiled.yaml index 5dd45c9c84f..20374ab81c1 100644 --- a/test_data/compiled-workflows/take_nap_compiled.yaml +++ b/test_data/compiled-workflows/take_nap_compiled.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: take-nap-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/take_nap_pipeline_root_compiled.yaml b/test_data/compiled-workflows/take_nap_pipeline_root_compiled.yaml index 5dd45c9c84f..20374ab81c1 100644 --- a/test_data/compiled-workflows/take_nap_pipeline_root_compiled.yaml +++ b/test_data/compiled-workflows/take_nap_pipeline_root_compiled.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: take-nap-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/two_step_pipeline.yaml b/test_data/compiled-workflows/two_step_pipeline.yaml index 94b3ca8e914..f5fa6c6a735 100644 --- a/test_data/compiled-workflows/two_step_pipeline.yaml +++ b/test_data/compiled-workflows/two_step_pipeline.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: simple-two-step-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/two_step_pipeline_containerized.yaml b/test_data/compiled-workflows/two_step_pipeline_containerized.yaml index a6739e86b03..04bd2608f1f 100644 --- a/test_data/compiled-workflows/two_step_pipeline_containerized.yaml +++ b/test_data/compiled-workflows/two_step_pipeline_containerized.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: containerized-two-step-pipeline- spec: arguments: diff --git a/test_data/compiled-workflows/upload_download_compiled.yaml b/test_data/compiled-workflows/upload_download_compiled.yaml index 67bc3a8b150..faee48c16a0 100644 --- a/test_data/compiled-workflows/upload_download_compiled.yaml +++ b/test_data/compiled-workflows/upload_download_compiled.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: test-data-passing-pipeline-1- spec: arguments: diff --git a/test_data/compiled-workflows/xgboost_sample_pipeline.yaml b/test_data/compiled-workflows/xgboost_sample_pipeline.yaml index 88199c14e89..b4fe9086b0f 100644 --- a/test_data/compiled-workflows/xgboost_sample_pipeline.yaml +++ b/test_data/compiled-workflows/xgboost_sample_pipeline.yaml @@ -1,7 +1,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - creationTimestamp: null generateName: xgboost-sample-pipeline- spec: arguments: From 33f2cad18a26dbefff97b39fff907e4906f45a77 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Fri, 27 Mar 2026 22:50:03 +0300 Subject: [PATCH 45/51] skip unit test: envs now defined in manifests, verified in existed workflow-compiler tests Signed-off-by: arpechenin --- backend/src/v2/compiler/argocompiler/container_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/src/v2/compiler/argocompiler/container_test.go b/backend/src/v2/compiler/argocompiler/container_test.go index db01e87c8cb..0388015dff5 100644 --- a/backend/src/v2/compiler/argocompiler/container_test.go +++ b/backend/src/v2/compiler/argocompiler/container_test.go @@ -70,7 +70,12 @@ func TestAddContainerExecutorTemplate(t *testing.T) { } +// With the transition from Container to Plugin, environment variable configuration +// is now defined in manifests rather than in code, so we can no longer verify +// their behavior in a unit test. However, this is still fully tested within the +// workflow-compiler integration tests. func TestContainerDriverTemplate_IncludesKFPPodNameEnv(t *testing.T) { + t.Skip("Skipping test: env vars are now defined in manifests and cannot be verified in unit tests") proxy.InitializeConfigWithEmptyForTests() c := &workflowCompiler{ templates: make(map[string]*wfapi.Template), From 11d59f28e84671ad74798a5ac721aacbdf6a70f8 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Sat, 28 Mar 2026 16:22:04 +0300 Subject: [PATCH 46/51] fix react driver logs query Signed-off-by: arpechenin --- frontend/src/components/tabs/RuntimeNodeDetailsV2.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/components/tabs/RuntimeNodeDetailsV2.tsx b/frontend/src/components/tabs/RuntimeNodeDetailsV2.tsx index c1faf348d37..f986823cb6e 100644 --- a/frontend/src/components/tabs/RuntimeNodeDetailsV2.tsx +++ b/frontend/src/components/tabs/RuntimeNodeDetailsV2.tsx @@ -170,7 +170,7 @@ function TaskNodeDetail({ }); const { data: driverLogsInfo } = useQuery({ - queryKey: queryKeys.executionLogs(execution?.getId(), namespace), + queryKey: queryKeys.driverLogs(execution?.getId(), namespace), queryFn: async (): Promise> => { if (!execution) { throw new Error('No execution is found.'); From ddf186d1720263146d1764211f57481ec1d8c628 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Sat, 28 Mar 2026 17:06:01 +0300 Subject: [PATCH 47/51] add kfp envs Signed-off-by: arpechenin --- .../base/pipeline/ml-pipeline-driver-plugin-cm.yaml | 8 ++++++++ .../patches/ml-pipeline-driver-plugin-cm.yaml | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml index 17756563d51..e88030a8740 100644 --- a/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml +++ b/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml @@ -11,6 +11,14 @@ data: image: ghcr.io/kubeflow/kfp-driver:dummy imagePullPolicy: IfNotPresent env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid - name: LOG_ACCESS_KEY valueFrom: secretKeyRef: diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-driver-plugin-cm.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-driver-plugin-cm.yaml index 02072786ecf..6f45d8b659a 100644 --- a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-driver-plugin-cm.yaml +++ b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-driver-plugin-cm.yaml @@ -10,6 +10,14 @@ data: name: driver-plugin image: ghcr.io/kubeflow/kfp-driver:dummy env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid - name: LOG_ACCESS_KEY valueFrom: secretKeyRef: From 91185346ea086913c9f7983e81e88784205ed717 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Sat, 28 Mar 2026 19:44:45 +0300 Subject: [PATCH 48/51] add roles nececcary rbac Signed-off-by: arpechenin --- .../pipelines-profile-controller/sync.py | 2 + .../base/pipeline/pipeline-runner-role.yaml | 46 ++++++++++++++++++- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py index 3016c0abfb6..bc7f0dc8374 100644 --- a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py +++ b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/sync.py @@ -240,6 +240,8 @@ def upsert_executor_plugin_sa(self, namespace): agent_sa_name = "ml-pipeline-driver-agent-executor-plugin" self.create_role_and_binding(rbac_v1=rbac_v1, namespace=namespace, role_name='configmap-reader', resources=["configmaps"], sa_name=agent_sa_name, verbs=["get", "list", "watch"]) + self.create_role_and_binding(rbac_v1=rbac_v1, namespace=namespace, role_name='ml-pipeline-driver-pods-reader', resources=["pods"], sa_name=agent_sa_name, verbs=["get", "list", "watch"]) + self.create_role_and_binding(rbac_v1=rbac_v1, namespace=namespace, role_name='ml-pipeline-driver-pvc-editor', resources=["persistentvolumeclaims"], sa_name=agent_sa_name, verbs=["create", "get", "list"]) self.create_role_and_binding(rbac_v1=rbac_v1, namespace=namespace, role_name='artifact-secret-reader', resources=["secrets"], sa_name=agent_sa_name, verbs=["get", "list", "watch"], resource_names=["mlpipeline-minio-artifact"]) except ApiException as e: if e.status == 409: diff --git a/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml b/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml index 23ad90b1bf2..7b6f3c2da96 100644 --- a/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml +++ b/manifests/kustomize/base/pipeline/pipeline-runner-role.yaml @@ -99,7 +99,50 @@ metadata: application-crd-id: kubeflow-pipelines secrets: - name: ml-pipeline-driver-agent-executor-plugin.service-account-token - +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ml-pipeline-driver-pods-viewer + namespace: kubeflow +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ml-pipeline-driver-pods-viewer-binding +subjects: + - kind: ServiceAccount + name: ml-pipeline-driver-agent-executor-plugin +roleRef: + kind: Role + name: ml-pipeline-driver-pods-viewer + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ml-pipeline-driver-pvc-editor + namespace: kubeflow +rules: + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["create", "get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ml-pipeline-driver-pvc-editor-binding +subjects: + - kind: ServiceAccount + name: ml-pipeline-driver-agent-executor-plugin +roleRef: + kind: Role + name: ml-pipeline-driver-pvc-editor + apiGroup: rbac.authorization.k8s.io --- apiVersion: v1 kind: Secret @@ -130,7 +173,6 @@ roleRef: kind: Role name: configmap-reader apiGroup: rbac.authorization.k8s.io - --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role From a85d3de6b9e1561b2d098b8251963b94a98e64bc Mon Sep 17 00:00:00 2001 From: arpechenin Date: Sun, 29 Mar 2026 10:27:33 +0300 Subject: [PATCH 49/51] add kfo envs to tests Signed-off-by: arpechenin --- .../standalone/tls-enabled/driver-plugin-cm-path.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml b/.github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml index 1f88b869662..47aabc4e103 100644 --- a/.github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml +++ b/.github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml @@ -10,6 +10,14 @@ data: ports: - containerPort: 8080 env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid - name: LOG_ACCESS_KEY valueFrom: secretKeyRef: From 70d33acab55ca3b3342f56d977599ca9a3a72aac Mon Sep 17 00:00:00 2001 From: arpechenin Date: Sun, 29 Mar 2026 10:29:12 +0300 Subject: [PATCH 50/51] add kfo envs to tests Signed-off-by: arpechenin --- .../manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml b/.github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml index 47aabc4e103..129e5ce2cdd 100644 --- a/.github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml +++ b/.github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml @@ -2,6 +2,8 @@ apiVersion: v1 kind: ConfigMap metadata: name: ml-pipeline-driver-agent + labels: + workflows.argoproj.io/configmap-type: ExecutorPlugin data: sidecar.container: | name: driver-plugin From 2e5b30c134b637170db3a5e64bfde5d59b9452d3 Mon Sep 17 00:00:00 2001 From: arpechenin Date: Sun, 29 Mar 2026 10:33:34 +0300 Subject: [PATCH 51/51] - fix plugin cm - add rbac to driver for pvc and pods - remove redundant enters Signed-off-by: arpechenin --- .../manifests/base/driver-plugin-cm-path.yaml | 2 -- .../standalone/tls-enabled/driver-plugin-cm-path.yaml | 11 ----------- backend/Makefile | 2 +- .../pipelines-profile-controller-admin.yaml | 6 ++++++ .../base/pipeline/ml-pipeline-driver-plugin-cm.yaml | 8 -------- .../patches/ml-pipeline-driver-plugin-cm.yaml | 10 ---------- .../kustomize/env/dev/driver-plugin-cm-path.yaml | 2 +- 7 files changed, 8 insertions(+), 33 deletions(-) diff --git a/.github/resources/manifests/base/driver-plugin-cm-path.yaml b/.github/resources/manifests/base/driver-plugin-cm-path.yaml index 08f446793e7..7da2bdd54b4 100644 --- a/.github/resources/manifests/base/driver-plugin-cm-path.yaml +++ b/.github/resources/manifests/base/driver-plugin-cm-path.yaml @@ -40,5 +40,3 @@ data: - name: var-run-argo mountPath: /kfp/log readOnly: false - - diff --git a/.github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml b/.github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml index 129e5ce2cdd..a37f6be0df6 100644 --- a/.github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml +++ b/.github/resources/manifests/standalone/tls-enabled/driver-plugin-cm-path.yaml @@ -2,8 +2,6 @@ apiVersion: v1 kind: ConfigMap metadata: name: ml-pipeline-driver-agent - labels: - workflows.argoproj.io/configmap-type: ExecutorPlugin data: sidecar.container: | name: driver-plugin @@ -12,14 +10,6 @@ data: ports: - containerPort: 8080 env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - name: LOG_ACCESS_KEY valueFrom: secretKeyRef: @@ -53,4 +43,3 @@ data: - name: var-run-argo mountPath: /kfp/log readOnly: false - diff --git a/backend/Makefile b/backend/Makefile index 15386e71191..ddae2ea9e47 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -95,7 +95,7 @@ kind-cluster-agnostic: kubectl wait --for condition=established --timeout=1m crd/applications.app.k8s.io # Deploy KFP @if [ "${TLS_ENABLED}" = "true" ]; then \ - kubectl apply -k $(CURDIR)/../manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls; \ + kubectl apply -k $(CURDIR)/../manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls; \ else \ kubectl apply -k $(CURDIR)/../manifests/kustomize/env/platform-agnostic; \ fi diff --git a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/pipelines-profile-controller-admin.yaml b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/pipelines-profile-controller-admin.yaml index c0cfd935df7..5dfcd3bc206 100644 --- a/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/pipelines-profile-controller-admin.yaml +++ b/manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/pipelines-profile-controller-admin.yaml @@ -21,6 +21,12 @@ rules: - apiGroups: [ "" ] resources: [ "secrets" ] verbs: [ "get", "list", "watch" ] + - apiGroups: [ "" ] + resources: [ "pods" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "" ] + resources: [ "persistentvolumeclaims" ] + verbs: [ "create", "get", "list" ] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml index e88030a8740..17756563d51 100644 --- a/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml +++ b/manifests/kustomize/base/pipeline/ml-pipeline-driver-plugin-cm.yaml @@ -11,14 +11,6 @@ data: image: ghcr.io/kubeflow/kfp-driver:dummy imagePullPolicy: IfNotPresent env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - name: LOG_ACCESS_KEY valueFrom: secretKeyRef: diff --git a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-driver-plugin-cm.yaml b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-driver-plugin-cm.yaml index 6f45d8b659a..6d88ee499e1 100644 --- a/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-driver-plugin-cm.yaml +++ b/manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls/patches/ml-pipeline-driver-plugin-cm.yaml @@ -2,22 +2,12 @@ apiVersion: v1 kind: ConfigMap metadata: name: ml-pipeline-driver-agent - labels: - workflows.argoproj.io/configmap-type: ExecutorPlugin data: sidecar.automountServiceAccountToken: "true" sidecar.container: | name: driver-plugin image: ghcr.io/kubeflow/kfp-driver:dummy env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - name: LOG_ACCESS_KEY valueFrom: secretKeyRef: diff --git a/manifests/kustomize/env/dev/driver-plugin-cm-path.yaml b/manifests/kustomize/env/dev/driver-plugin-cm-path.yaml index f6918cec176..02e10ab683e 100644 --- a/manifests/kustomize/env/dev/driver-plugin-cm-path.yaml +++ b/manifests/kustomize/env/dev/driver-plugin-cm-path.yaml @@ -38,4 +38,4 @@ data: volumeMounts: - name: var-run-argo mountPath: /kfp/log - readOnly: false \ No newline at end of file + readOnly: false